pixellint-core 0.59.0

//! Vendor endpoint directory.
//!
//! The directory answers one question a rulepack cannot: whose pixel is this?
//! It maps hosts to vendors and nothing else. Entries carry no parameter
//! contracts and no rule text, so a directory hit never asserts that an
//! artifact is right or wrong; it says which vendor owns the endpoint and
//! whether Pixellint has a rulepack for it.
//!
//! That split is deliberate. Rules require a citable contract, and most vendors
//! never publish one. Attribution is a weaker claim that can be made honestly
//! for the long tail.

use std::borrow::Cow;
use std::collections::HashMap;
use std::error::Error;
use std::fmt;
use std::fs;
use std::path::Path;

use serde::{Deserialize, Serialize};

/// Pseudo-rulepack id used to include or exclude the directory in
/// [`ValidationOptions`](crate::ValidationOptions).
pub const DIRECTORY_ID: &str = "directory";

/// One vendor and the endpoints it serves.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct VendorEntry {
    /// Stable vendor slug, reported as `detected_vendor`.
    pub vendor: String,
    pub display_name: String,
    /// Broad function of the endpoint: social, programmatic, analytics, and so on.
    pub category: String,
    /// Hosts the vendor serves. A host also matches its subdomains.
    pub hosts: Vec<String>,
    /// Optional path restrictions for services on a shared host. Host-only
    /// lookup never attributes an entry restricted to these prefixes.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub path_prefixes: Vec<String>,
    /// First-party rulepack that covers some of this vendor's endpoints, if one
    /// exists.
    #[serde(default)]
    pub rulepack: Option<String>,
}

/// A set of vendor entries.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct VendorDirectory {
    entries: Vec<VendorEntry>,
    #[serde(skip)]
    by_host: HashMap<String, usize>,
}

impl PartialEq for VendorDirectory {
    fn eq(&self, other: &Self) -> bool {
        self.entries == other.entries
    }
}

impl Eq for VendorDirectory {}

/// Why a directory could not be loaded.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DirectoryError {
    Parse(String),
    Io(String),
    EmptyField {
        vendor: String,
        field: &'static str,
    },
    DuplicateHost {
        host: String,
        vendors: (String, String),
    },
    InvalidPathPrefix {
        vendor: String,
        prefix: String,
    },
}

impl fmt::Display for DirectoryError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::Parse(error) => write!(f, "invalid vendor directory JSON: {error}"),
            Self::Io(error) => write!(f, "could not read vendor directory: {error}"),
            Self::EmptyField { vendor, field } => {
                write!(
                    f,
                    "vendor directory entry `{vendor}` has an empty `{field}`"
                )
            }
            Self::DuplicateHost {
                host,
                vendors: (first, second),
            } => write!(
                f,
                "host `{host}` is claimed by both `{first}` and `{second}` in the vendor directory"
            ),
            Self::InvalidPathPrefix { vendor, prefix } => write!(
                f,
                "vendor directory entry `{vendor}` has an invalid path prefix `{prefix}`"
            ),
        }
    }
}

impl Error for DirectoryError {}

impl VendorDirectory {
    /// The directory compiled into the crate.
    ///
    /// Parsing is proven by tests, so a malformed built-in directory is a build
    /// failure rather than a runtime surprise.
    pub fn builtin() -> Self {
        Self::from_json(crate::BUILTIN_VENDOR_DIRECTORY)
            .expect("built-in vendor directory should be valid")
    }

    pub fn from_json(json: &str) -> Result<Self, DirectoryError> {
        let mut directory: Self =
            serde_json::from_str(json).map_err(|error| DirectoryError::Parse(error.to_string()))?;
        directory.validate()?;
        directory.rebuild_host_index();
        Ok(directory)
    }

    pub fn from_path(path: impl AsRef<Path>) -> Result<Self, DirectoryError> {
        let path = path.as_ref();
        let json = fs::read_to_string(path)
            .map_err(|error| DirectoryError::Io(format!("{}: {error}", path.display())))?;
        Self::from_json(&json)
    }

    fn validate(&self) -> Result<(), DirectoryError> {
        let mut claimed: Vec<(Cow<'_, str>, &str)> = Vec::new();

        for entry in &self.entries {
            for (field, value) in [
                ("vendor", &entry.vendor),
                ("display_name", &entry.display_name),
                ("category", &entry.category),
            ] {
                if value.trim().is_empty() {
                    return Err(DirectoryError::EmptyField {
                        vendor: entry.vendor.clone(),
                        field,
                    });
                }
            }

            if entry.hosts.is_empty() {
                return Err(DirectoryError::EmptyField {
                    vendor: entry.vendor.clone(),
                    field: "hosts",
                });
            }

            for prefix in &entry.path_prefixes {
                if !prefix.starts_with('/') || prefix.contains(['?', '#']) {
                    return Err(DirectoryError::InvalidPathPrefix {
                        vendor: entry.vendor.clone(),
                        prefix: prefix.clone(),
                    });
                }
            }

            for host in &entry.hosts {
                let host = ascii_lower(host);

                if let Some((_, owner)) = claimed
                    .iter()
                    .find(|(claimed, _)| claimed.as_ref() == host.as_ref())
                {
                    return Err(DirectoryError::DuplicateHost {
                        host: host.into_owned(),
                        vendors: ((*owner).to_string(), entry.vendor.clone()),
                    });
                }

                claimed.push((host, entry.vendor.as_str()));
            }
        }

        Ok(())
    }

    /// Adds entries from another directory. A host already claimed by a
    /// different vendor is rejected, so an overlay cannot steal attribution.
    pub fn merge(&mut self, other: Self) -> Result<(), DirectoryError> {
        self.entries.extend(other.entries);
        self.validate()?;
        self.rebuild_host_index();
        Ok(())
    }

    pub fn entries(&self) -> &[VendorEntry] {
        &self.entries
    }

    pub fn len(&self) -> usize {
        self.entries.len()
    }

    pub fn is_empty(&self) -> bool {
        self.entries.is_empty()
    }

    /// Total hosts across every entry.
    pub fn host_count(&self) -> usize {
        self.entries.iter().map(|entry| entry.hosts.len()).sum()
    }

    /// Finds the vendor that serves a host. A directory host also matches its
    /// subdomains, so `sc-static.net` covers `cdn.sc-static.net`.
    ///
    /// When more than one listed host matches, the earlier directory entry
    /// wins, matching the previous linear scan.
    ///
    /// Prepared hosts are already lowercased. This path skips a second copy in
    /// that case.
    pub fn lookup_host(&self, host: &str) -> Option<&VendorEntry> {
        self.lookup(host, None)
    }

    /// Attributes a parsed host and path. Prefixes are case-sensitive and
    /// inspect the path only, never a query string or fragment.
    pub fn lookup_endpoint(&self, host: &str, path: &str) -> Option<&VendorEntry> {
        // PreparedArtifact preserves raw paths to retain source spans. Match
        // attribution on the path a browser requests after URL normalization.
        let canonical = path
            .starts_with('/')
            .then(|| url::Url::parse(&format!("https://directory.invalid{path}")))
            .and_then(Result::ok);
        self.lookup(host, canonical.as_ref().map(url::Url::path))
    }

    fn lookup(&self, host: &str, path: Option<&str>) -> Option<&VendorEntry> {
        if host.bytes().any(|byte| byte.is_ascii_uppercase()) {
            self.lookup_lowered(&host.to_ascii_lowercase(), path)
        } else {
            self.lookup_lowered(host, path)
        }
    }

    fn lookup_lowered(&self, host: &str, path: Option<&str>) -> Option<&VendorEntry> {
        let mut best: Option<usize> = None;
        let mut label = host;
        loop {
            if let Some(&index) = self.by_host.get(label) {
                let entry = &self.entries[index];
                let matches_path = entry.path_prefixes.is_empty()
                    || path.is_some_and(|path| {
                        entry
                            .path_prefixes
                            .iter()
                            .any(|prefix| path.starts_with(prefix))
                    });
                if matches_path {
                    if index == 0 {
                        return Some(entry);
                    }
                    best = Some(best.map_or(index, |current| current.min(index)));
                }
            }
            match label.find('.') {
                Some(dot) => label = &label[dot + 1..],
                None => break,
            }
        }
        best.map(|index| &self.entries[index])
    }

    fn rebuild_host_index(&mut self) {
        for entry in &mut self.entries {
            for host in &mut entry.hosts {
                if host.bytes().any(|byte| byte.is_ascii_uppercase()) {
                    host.make_ascii_lowercase();
                }
            }
        }
        let mut by_host = HashMap::with_capacity(self.host_count());
        for (index, entry) in self.entries.iter().enumerate() {
            for host in &entry.hosts {
                by_host.entry(host.clone()).or_insert(index);
            }
        }
        self.by_host = by_host;
    }
}

fn ascii_lower(value: &str) -> Cow<'_, str> {
    if value.bytes().any(|byte| byte.is_ascii_uppercase()) {
        Cow::Owned(value.to_ascii_lowercase())
    } else {
        Cow::Borrowed(value)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    const TEST_DIRECTORY: &str = r#"{
        "entries": [
            {
                "vendor": "acme",
                "display_name": "Acme",
                "category": "programmatic",
                "hosts": ["px.acme.example", "acme-static.example"],
                "rulepack": "vendor/acme"
            },
            {
                "vendor": "globex",
                "display_name": "Globex",
                "category": "analytics",
                "hosts": ["globex.example"]
            }
        ]
    }"#;

    #[test]
    fn lookup_matches_exact_hosts_and_subdomains() {
        let directory = VendorDirectory::from_json(TEST_DIRECTORY).expect("parse");

        assert_eq!(
            directory.lookup_host("px.acme.example").map(|e| &e.vendor),
            Some(&"acme".to_string())
        );
        assert_eq!(
            directory
                .lookup_host("cdn.acme-static.example")
                .map(|e| &e.vendor),
            Some(&"acme".to_string())
        );
        assert_eq!(
            directory.lookup_host("GLOBEX.EXAMPLE").map(|e| &e.vendor),
            Some(&"globex".to_string())
        );
        assert_eq!(directory.lookup_host("notglobex.example"), None);
        assert_eq!(directory.lookup_host("example.com"), None);
    }

    #[test]
    fn overlapping_suffixes_keep_the_first_directory_entry() {
        let directory = VendorDirectory::from_json(
            r#"{
                "entries": [
                    {
                        "vendor": "parent",
                        "display_name": "Parent",
                        "category": "analytics",
                        "hosts": ["example.test"]
                    },
                    {
                        "vendor": "child",
                        "display_name": "Child",
                        "category": "analytics",
                        "hosts": ["px.example.test"]
                    }
                ]
            }"#,
        )
        .expect("parse");

        assert_eq!(
            directory.lookup_host("px.example.test").map(|e| &e.vendor),
            Some(&"parent".to_string())
        );
        assert_eq!(
            directory
                .lookup_host("cdn.px.example.test")
                .map(|e| &e.vendor),
            Some(&"parent".to_string())
        );
        assert_eq!(
            directory.lookup_host("example.test").map(|e| &e.vendor),
            Some(&"parent".to_string())
        );
    }

    #[test]
    fn duplicate_hosts_are_rejected() {
        let json = TEST_DIRECTORY.replace(r#""globex.example""#, r#""px.acme.example""#);
        let error = VendorDirectory::from_json(&json).expect_err("duplicates are caught");
        assert!(
            matches!(error, DirectoryError::DuplicateHost { .. }),
            "{error}"
        );
    }

    #[test]
    fn merge_adds_new_hosts_and_rejects_stolen_ones() {
        let mut directory = VendorDirectory::from_json(TEST_DIRECTORY).expect("parse");
        let extra = VendorDirectory::from_json(
            r#"{
                "entries": [
                    {
                        "vendor": "initech",
                        "display_name": "Initech",
                        "category": "analytics",
                        "hosts": ["px.initech.example"]
                    }
                ]
            }"#,
        )
        .expect("overlay");
        directory.merge(extra).expect("new hosts merge");
        assert_eq!(
            directory
                .lookup_host("px.initech.example")
                .map(|entry| entry.vendor.as_str()),
            Some("initech")
        );

        let stolen = VendorDirectory::from_json(
            r#"{
                "entries": [
                    {
                        "vendor": "initech",
                        "display_name": "Initech",
                        "category": "analytics",
                        "hosts": ["px.acme.example"]
                    }
                ]
            }"#,
        )
        .expect("stolen overlay");
        let error = directory.merge(stolen).expect_err("cannot steal a host");
        assert!(
            matches!(error, DirectoryError::DuplicateHost { .. }),
            "{error}"
        );
    }

    #[test]
    fn entries_need_a_vendor_and_hosts() {
        let json = TEST_DIRECTORY.replace(r#""vendor": "globex""#, r#""vendor": """#);
        let error = VendorDirectory::from_json(&json).expect_err("empty vendor is caught");
        assert!(
            matches!(error, DirectoryError::EmptyField { .. }),
            "{error}"
        );

        let json = TEST_DIRECTORY.replace(r#""hosts": ["globex.example"]"#, r#""hosts": []"#);
        let error = VendorDirectory::from_json(&json).expect_err("empty hosts are caught");
        assert!(
            matches!(error, DirectoryError::EmptyField { .. }),
            "{error}"
        );
    }

    #[test]
    fn unknown_fields_are_rejected() {
        let json =
            TEST_DIRECTORY.replace(r#""vendor": "acme","#, r#""vendor": "acme", "oops": 1,"#);
        let error = VendorDirectory::from_json(&json).expect_err("unknown fields are caught");
        assert!(matches!(error, DirectoryError::Parse(_)), "{error}");
    }

    #[test]
    fn the_builtin_directory_loads_and_covers_the_shipped_packs() {
        let directory = VendorDirectory::builtin();

        assert!(
            directory.len() >= 80,
            "directory shrank to {}",
            directory.len()
        );
        assert!(directory.host_count() >= 200);

        for host in [
            "www.facebook.com",
            "analytics.tiktok.com",
            "ct.pinterest.com",
            "trc.taboola.com",
            "pixel.quantserve.com",
            "track.celtra.com",
            "tpsc-video-as.doubleverify.com",
            "tpsc-video-eu.doubleverify.com",
            "d9.flashtalking.com",
            "qa-xre.flashtalking.net",
            "s.update.3lift.com",
            "rtb-us-west.linkedin.com",
            "beacons.extremereach.io",
            "analytics.adcanvas.com",
            "stats.sxp.smartclip.net",
            "log.xpln.tech",
            "cs10.connected-stories.com",
            "postback.iqm.com",
            "tr.blismedia.com",
            "www.googletagmanager.com",
            "ad.doubleclick.net",
            "ade.googlesyndication.com",
            "pubads.g.doubleclick.net",
            "vfw.amazon-adsystem.com",
            "s.amazon-adsystem.com",
            "e-11428.adzerk.net",
            "eu-adsrv.rtbsuperhub.com",
            "event.havasedge.com",
            "beeswax-ipv4-prod.telemetry.vaultdcr.com",
        ] {
            assert!(
                directory.lookup_host(host).is_some(),
                "{host} is not in the directory"
            );
        }

        let gtm = directory
            .lookup_host("www.googletagmanager.com")
            .expect("gtm");
        assert_eq!(gtm.rulepack.as_deref(), Some("vendor/google-tag-manager"));

        let cm360 = directory.lookup_host("ad.doubleclick.net").expect("cm360");
        assert_eq!(cm360.rulepack.as_deref(), Some("vendor/floodlight"));

        let gam = directory
            .lookup_host("ade.googlesyndication.com")
            .expect("gam");
        assert_eq!(gam.display_name, "Google Ad Manager");
        assert_eq!(gam.rulepack.as_deref(), Some("vendor/google-ad-manager"));

        let firefly = directory
            .lookup_host("vfw.amazon-adsystem.com")
            .expect("vfw");
        assert_eq!(firefly.rulepack.as_deref(), Some("vendor/amazon-vfw"));

        let ad_tag = directory
            .lookup_host("s.amazon-adsystem.com")
            .expect("ad tag");
        assert_eq!(ad_tag.rulepack.as_deref(), Some("vendor/amazon-ads"));

        // Every rulepack an entry points at has to exist, and every first-party
        // pack's vendor has to point at some pack. Directory hits on a covered
        // vendor should say coverage exists elsewhere, not that the vendor is
        // unknown to the rulepacks. A vendor may have several directory rows
        // (Google Tag Manager vs Google Ad Manager); only one of them needs a
        // pointer.
        let pack_ids: Vec<&str> = crate::BUILTIN_VENDOR_MANIFESTS
            .iter()
            .map(|(id, _)| *id)
            .collect();
        for entry in directory.entries() {
            if let Some(rulepack) = &entry.rulepack {
                assert!(
                    pack_ids.contains(&rulepack.as_str()),
                    "directory entry `{}` points at unknown rulepack `{rulepack}`",
                    entry.vendor
                );
            }
        }

        for (id, json) in crate::BUILTIN_VENDOR_MANIFESTS {
            let manifest: serde_json::Value =
                serde_json::from_str(json).unwrap_or_else(|error| panic!("{id}: {error}"));
            let vendor = manifest["vendor"]
                .as_str()
                .unwrap_or_else(|| panic!("{id} is missing vendor"));
            let entries: Vec<_> = directory
                .entries()
                .iter()
                .filter(|entry| entry.vendor == vendor)
                .collect();
            assert!(
                !entries.is_empty(),
                "no directory entry for pack `{id}` vendor `{vendor}`"
            );
            assert!(
                entries.iter().any(|entry| entry.rulepack.is_some()),
                "directory vendor `{vendor}` has pack `{id}` but no rulepack pointer"
            );
        }
    }
}