leyline-http 0.1.5

HTTP client that impersonates Chrome, Firefox, and Safari on the wire: TLS (JA3/JA4), HTTP/2, and HTTP/3 fingerprints
Documentation
use std::collections::{BTreeMap, HashSet};
use std::fmt::Write as _;
use std::fs;
use std::path::{Path, PathBuf};

use serde::Deserialize;

use crate::{BuildResult, is_ident};

#[path = "../src/profile/browser/digest.rs"]
mod digest;

const SHARED_DATA: &[&str] = &["bare.toml", "brands.toml", "headers.toml", "platforms.toml"];

#[derive(Deserialize)]
pub(crate) struct Families {
    family: Vec<FamilyRow>,
}

#[derive(Deserialize)]
struct FamilyRow {
    variant: String,
    label: String,
    browsers: Vec<String>,
    #[serde(default)]
    default: bool,
    #[serde(default = "browser_capture")]
    latest_capture: Vec<Capture>,
    #[serde(default)]
    samesite_redirect_chain: bool,
}

fn browser_capture() -> Vec<Capture> {
    vec![Capture::Browser]
}

#[derive(Deserialize)]
struct ProfileFile {
    meta: Meta,
}

#[derive(Deserialize, Clone, Copy, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
enum Capture {
    Browser,
    Native,
    HeadlessShell,
    Webview,
    Emulator,
    Inferred,
    SelfReferential,
}

#[derive(Deserialize)]
pub(crate) struct Meta {
    name: String,
    browser: String,
    version: u32,
    capture: Capture,
    pub(crate) variant: String,
    #[serde(default)]
    hello: Option<u32>,
    #[serde(default)]
    platform_browser: BTreeMap<String, String>,
    #[serde(default)]
    deprecated: Option<String>,
    #[serde(default)]
    pub(crate) header_style: Option<String>,
}

pub(crate) struct Row {
    pub(crate) meta: Meta,
    pub(crate) path: String,
    pub(crate) id: u32,
    family: usize,
    digest: u64,
}

fn shared_data(root: &Path) -> BuildResult<Vec<String>> {
    println!("cargo:rerun-if-changed=src/profile/browser/digest.rs");
    SHARED_DATA
        .iter()
        .map(|name| Ok(fs::read_to_string(root.join(name))?))
        .collect()
}

fn family_id(family: &FamilyRow) -> String {
    family.label.to_ascii_lowercase().replace(' ', "-")
}

fn is_id(id: &str) -> bool {
    !id.is_empty()
        && id
            .chars()
            .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-')
}

pub(crate) fn load_rows(root: &Path, families: &Families) -> BuildResult<Vec<Row>> {
    let shared = shared_data(root)?;
    let mut rows = Vec::new();
    for dir in sorted(root)? {
        if !dir.is_dir() {
            continue;
        }
        for file in sorted(&dir)? {
            if file.extension().is_none_or(|ext| ext != "toml") {
                continue;
            }
            let text = fs::read_to_string(&file)?;
            let meta = toml::from_str::<ProfileFile>(&text)
                .map_err(|e| format!("{}: {e}", file.display()))?
                .meta;
            let family = families
                .family
                .iter()
                .position(|f| f.browsers.contains(&meta.browser))
                .ok_or_else(|| {
                    format!(
                        "{}: browser {:?} is in no families.toml entry",
                        file.display(),
                        meta.browser
                    )
                })?;
            let path = file
                .strip_prefix(root)?
                .to_str()
                .ok_or("profile path is not UTF-8")?
                .replace('\\', "/");
            let mut parts: Vec<&[u8]> = shared.iter().map(String::as_bytes).collect();
            parts.push(text.as_bytes());
            let digest = digest::digest(&parts);
            rows.push(Row {
                meta,
                path,
                id: 0,
                family,
                digest,
            });
        }
    }
    rows.sort_by(|a, b| {
        (a.family, &a.meta.browser, a.meta.version).cmp(&(
            b.family,
            &b.meta.browser,
            b.meta.version,
        ))
    });
    Ok(rows)
}

fn sorted(dir: &Path) -> BuildResult<Vec<PathBuf>> {
    let mut paths = Vec::new();
    for entry in fs::read_dir(dir)? {
        paths.push(entry?.path());
    }
    paths.sort();
    Ok(paths)
}

pub(crate) fn check_unique(rows: &[Row], families: &Families) -> BuildResult<()> {
    let mut variants = HashSet::new();
    let mut keys = HashSet::new();
    for row in rows {
        if !is_ident(&row.meta.variant) {
            return Err(format!(
                "{}: variant {:?} is not a Rust type name",
                row.path, row.meta.variant
            )
            .into());
        }
        if !variants.insert(row.meta.variant.as_str()) {
            return Err(format!(
                "{}: variant {} is declared twice",
                row.path, row.meta.variant
            )
            .into());
        }
        if !is_id(&row.meta.browser) {
            return Err(format!(
                "{}: browser {:?} is not a lowercase id",
                row.path, row.meta.browser
            )
            .into());
        }
        if !keys.insert((row.meta.browser.as_str(), row.meta.version)) {
            return Err(format!(
                "{}: {} {} is declared twice",
                row.path, row.meta.browser, row.meta.version
            )
            .into());
        }
    }
    let mut family_names = HashSet::new();
    let mut family_ids = HashSet::new();
    for family in &families.family {
        let id = family_id(family);
        if !is_ident(&family.variant)
            || !family_names.insert(family.variant.as_str())
            || !is_id(&id)
            || !family_ids.insert(id)
        {
            return Err(format!(
                "families.toml: bad or repeated variant {:?}",
                family.variant
            )
            .into());
        }
    }
    if families.family.iter().filter(|f| f.default).count() != 1 {
        return Err("families.toml: exactly one family needs default = true".into());
    }
    Ok(())
}

fn find<'a>(rows: &'a [Row], browser: &str, version: u32) -> BuildResult<&'a Row> {
    rows.iter()
        .find(|r| r.meta.browser == browser && r.meta.version == version)
        .ok_or_else(|| format!("no profile for {browser} {version}").into())
}

fn latest(rows: &[Row], keep: impl Fn(&Row) -> bool) -> Option<&Row> {
    rows.iter()
        .filter(|r| r.meta.deprecated.is_none() && keep(r))
        .max_by_key(|r| r.meta.version)
}

fn hello<'a>(rows: &'a [Row], row: &Row) -> BuildResult<&'a Row> {
    find(
        rows,
        &row.meta.browser,
        row.meta.hello.unwrap_or(row.meta.version),
    )
}

fn variant_list<'a>(rows: impl IntoIterator<Item = &'a Row>) -> String {
    rows.into_iter()
        .map(|r| format!("Browser::{}", r.meta.variant))
        .collect::<Vec<_>>()
        .join(", ")
}

pub(crate) fn render(rows: &[Row], families: &Families) -> BuildResult<String> {
    let mut out = String::new();
    writeln!(
        out,
        "#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]\n#[non_exhaustive]\npub enum Family {{"
    )?;
    for family in &families.family {
        writeln!(out, "    {},", family.variant)?;
    }
    writeln!(out, "}}\n")?;
    writeln!(
        out,
        "#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]\n#[non_exhaustive]\npub enum Browser {{"
    )?;
    for row in crate::ids::by_id(rows) {
        if let Some(note) = &row.meta.deprecated {
            writeln!(out, "    #[deprecated(note = {note:?})]")?;
        }
        writeln!(out, "    {},", row.meta.variant)?;
    }
    writeln!(out, "}}\n")?;

    let labels: Vec<String> = families
        .family
        .iter()
        .map(|f| format!("{:?}", f.label))
        .collect();
    writeln!(
        out,
        "const FAMILY_LABELS: &[&str] = &[{}];",
        labels.join(", ")
    )?;
    let ids: Vec<String> = families
        .family
        .iter()
        .map(|f| format!("{:?}", family_id(f)))
        .collect();
    writeln!(out, "const FAMILY_IDS: &[&str] = &[{}];", ids.join(", "))?;
    let all: Vec<String> = families
        .family
        .iter()
        .map(|f| format!("Family::{}", f.variant))
        .collect();
    writeln!(out, "const FAMILY_ALL: &[Family] = &[{}];", all.join(", "))?;
    let redirect_chain: Vec<String> = families
        .family
        .iter()
        .map(|f| f.samesite_redirect_chain.to_string())
        .collect();
    writeln!(
        out,
        "const FAMILY_SAMESITE_REDIRECT_CHAIN: &[bool] = &[{}];",
        redirect_chain.join(", ")
    )?;
    let mut family_latest = Vec::new();
    for (index, family) in families.family.iter().enumerate() {
        let row = latest(rows, |r| {
            r.family == index && family.latest_capture.contains(&r.meta.capture)
        })
        .ok_or_else(|| {
            format!(
                "families.toml: family {} has no profile whose capture is in latest_capture",
                family.variant
            )
        })?;
        family_latest.push(row);
    }
    writeln!(
        out,
        "#[allow(deprecated, reason = \"tables list every browser, including deprecated ones\")]\nconst FAMILY_LATEST: &[Browser] = &[{}];",
        variant_list(family_latest.iter().copied())
    )?;
    let default = families
        .family
        .iter()
        .find(|f| f.default)
        .ok_or("no default family")?;
    writeln!(
        out,
        "const DEFAULT_FAMILY: Family = Family::{};",
        default.variant
    )?;
    writeln!(
        out,
        "#[allow(deprecated, reason = \"tables list every browser, including deprecated ones\")]\nconst ALL: &[Browser] = &[{}];",
        variant_list(rows)
    )?;

    writeln!(
        out,
        "#[allow(deprecated, reason = \"tables list every browser, including deprecated ones\")]\nconst ENTRIES: &[Entry] = &["
    )?;
    for row in crate::ids::by_id(rows) {
        let rep = hello(rows, row)?;
        let mut platforms = Vec::new();
        for (platform, browser) in &row.meta.platform_browser {
            let target = latest(rows, |r| {
                &r.meta.browser == browser
                    && families.family[r.family]
                        .latest_capture
                        .contains(&r.meta.capture)
            })
            .ok_or_else(|| {
                format!(
                    "{}: platform_browser names {browser:?}, which has no profile whose capture \
                     is in its family's latest_capture",
                    row.path
                )
            })?;
            platforms.push(format!("({platform:?}, Browser::{})", target.meta.variant));
        }
        writeln!(
            out,
            "    Entry {{ family: Family::{}, key: {:?}, version: {}, id: \"{}-{}\", name: {:?}, \
             hello: Browser::{}, platforms: &[{}], digest: {:#018x}, \
             source: include_str!(concat!(env!(\"CARGO_MANIFEST_DIR\"), \"/profiles/{}\")) }},",
            families.family[row.family].variant,
            row.meta.browser,
            row.meta.version,
            row.meta.browser,
            row.meta.version,
            row.meta.name,
            rep.meta.variant,
            platforms.join(", "),
            row.digest,
            row.path,
        )?;
    }
    writeln!(out, "];")?;
    Ok(out)
}