spider 2.52.7

use crate::CaseInsensitiveString;

lazy_static! {
    static ref RE: regex::Regex = {
        unsafe {
            regex::Regex::new(
                r"(?x)
                    # list
                    (?<list>\{(?<items>[^}\\}^\{]+)}) |
                    # range
                    (?<range>\[(?:(?<start>(?<padding>0*)\d+|[a-z]))-(?:(?<end>\d+|[a-z]))(?::(?<step>\d+))?])
                ",
            )
            .unwrap_unchecked()
        }
    };
}

/// Optional cap on the number of URLs a single glob pattern may expand to, read
/// from `SPIDER_GLOB_MAX_EXPANSION`. Unset or `0` = unlimited (byte-identical to
/// the historical behaviour); a positive value refuses to materialize a larger
/// expansion so a crafted range (e.g. `[0-2000000000]`) can't OOM before crawling.
fn glob_max_expansion() -> Option<usize> {
    match std::env::var("SPIDER_GLOB_MAX_EXPANSION")
        .ok()
        .and_then(|v| v.parse::<usize>().ok())
    {
        None | Some(0) => None,
        Some(n) => Some(n),
    }
}

/// expand a website url to a glob pattern set
pub fn expand_url(url: &str) -> Vec<CaseInsensitiveString> {
    use itertools::Itertools;

    let mut matches = Vec::new();

    for capture in RE.captures_iter(url) {
        match (
            capture.name("list"),
            capture.name("items"),
            capture.name("range"),
            capture.name("start"),
            capture.name("end"),
        ) {
            // matches a list
            (Some(list), Some(items), _, _, _) => {
                let substring = list.as_str();

                let items = items
                    .as_str()
                    .split(",")
                    .map(|item| (item.to_string(), substring))
                    .collect::<Vec<(String, &str)>>();

                matches.push(items);
            }
            // matches a range
            (_, _, Some(range), Some(start), Some(end)) => {
                let substring = range.as_str();
                let step = match capture.name("step") {
                    // A malformed or overflowing step must not panic; fall back
                    // to 1. `.max(1)` guarantees `step_by` never receives 0
                    // (which panics with "step must be non-zero").
                    Some(step) => step.as_str().parse::<usize>().unwrap_or(1).max(1),
                    None => 1,
                };
                let start_str = start.as_str();
                let end_str = end.as_str();

                let width = match capture.name("padding") {
                    Some(padding) => {
                        if !padding.as_str().is_empty() {
                            start_str.len()
                        } else {
                            0
                        }
                    }
                    None => 0,
                };

                match (start_str.parse::<u32>(), end_str.parse::<u32>()) {
                    // start and end are numbers
                    (Ok(s), Ok(e)) => {
                        // `e + 1` overflows when `e == u32::MAX`. Compute the
                        // exclusive end with checked_add and fall back to an empty
                        // range on overflow — byte-identical to the prior
                        // release-mode wrap-to-empty, but without the debug panic.
                        let items = match e.checked_add(1) {
                            Some(end) => (s..end)
                                .step_by(step)
                                .map(|num| {
                                    (
                                        format!("{:0>width$}", num.to_string(), width = width),
                                        substring,
                                    )
                                })
                                .collect::<Vec<(String, &str)>>(),
                            None => Vec::new(),
                        };

                        matches.push(items);
                    }
                    // start and end are characters
                    _ => {
                        let s = start_str.as_bytes()[0];
                        let e = end_str.as_bytes()[0];
                        let items = (s..e + 1)
                            .map(|b| (String::from(b as char), substring))
                            .collect::<Vec<(String, &str)>>();

                        matches.push(items);
                    }
                };
            }
            _ => {}
        }
    }

    if matches.is_empty() {
        return Vec::new();
    }

    // Optional expansion cap (see `glob_max_expansion`). Unset/0 => skipped =>
    // byte-identical. `checked_mul` keeps the size computation itself from
    // overflowing; an overflow is treated as "over cap".
    if let Some(max) = glob_max_expansion() {
        let total = matches
            .iter()
            .try_fold(1usize, |acc, m| acc.checked_mul(m.len()));
        if total.map_or(true, |n| n > max) {
            log::warn!(
                "glob expansion of {:?} exceeds SPIDER_GLOB_MAX_EXPANSION={}; skipping expansion",
                url,
                max
            );
            return Vec::new();
        }
    }

    matches
        .iter()
        .multi_cartesian_product()
        .map(|combination| {
            let mut new_url = CaseInsensitiveString::new(url);

            for (replacement, substring) in combination {
                new_url = new_url
                    .inner()
                    .replace(substring, replacement.as_str())
                    .into();
            }

            new_url
        })
        .collect::<Vec<CaseInsensitiveString>>()
}

#[test]
fn test_expand_url_list() {
    let url = "https://choosealicense.com/licenses/{mit,apache-2.0,mpl-2.0}/";

    assert_eq!(
        expand_url(url),
        [
            "https://choosealicense.com/licenses/mit/",
            "https://choosealicense.com/licenses/apache-2.0/",
            "https://choosealicense.com/licenses/mpl-2.0/"
        ]
        .iter()
        .map(CaseInsensitiveString::new)
        .collect::<Vec<CaseInsensitiveString>>()
    );
}

#[test]
fn test_expand_url_list_escaped_closing() {
    let url = "https://choosealicense.com/licenses/{mit\\}/";

    assert_eq!(expand_url(url), Vec::<CaseInsensitiveString>::new());
}

#[test]
fn test_expand_url_numerical_range() {
    let url = "https://choosealicense.com/licenses/bsd-[2-4]-clause/";

    assert_eq!(
        expand_url(url),
        [
            "https://choosealicense.com/licenses/bsd-2-clause/",
            "https://choosealicense.com/licenses/bsd-3-clause/",
            "https://choosealicense.com/licenses/bsd-4-clause/",
        ]
        .iter()
        .map(CaseInsensitiveString::new)
        .collect::<Vec<CaseInsensitiveString>>()
    );
}

#[test]
fn test_expand_url_numerical_range_singe_item() {
    let url = "https://choosealicense.com/licenses/bsd-[4-4]-clause/";

    assert_eq!(
        expand_url(url),
        ["https://choosealicense.com/licenses/bsd-4-clause/"]
            .iter()
            .map(CaseInsensitiveString::new)
            .collect::<Vec<CaseInsensitiveString>>()
    );
}

#[test]
fn test_expand_url_numerical_range_with_step() {
    let url = "https://choosealicense.com/licenses/bsd-[2-4:2]-clause/";

    assert_eq!(
        expand_url(url),
        [
            "https://choosealicense.com/licenses/bsd-2-clause/",
            "https://choosealicense.com/licenses/bsd-4-clause/",
        ]
        .iter()
        .map(CaseInsensitiveString::new)
        .collect::<Vec<CaseInsensitiveString>>()
    );
}

#[test]
fn test_expand_url_numerical_range_with_padding() {
    let url = "https://choosealicense.com/licenses/bsd-[002-004]-clause/";

    assert_eq!(
        expand_url(url),
        [
            "https://choosealicense.com/licenses/bsd-002-clause/",
            "https://choosealicense.com/licenses/bsd-003-clause/",
            "https://choosealicense.com/licenses/bsd-004-clause/",
        ]
        .iter()
        .map(CaseInsensitiveString::new)
        .collect::<Vec<CaseInsensitiveString>>()
    );
}

#[test]
fn test_expand_url_numerical_range_with_padding_ignore_end_padding() {
    let url = "https://choosealicense.com/licenses/bsd-[008-10]-clause/";

    assert_eq!(
        expand_url(url),
        [
            "https://choosealicense.com/licenses/bsd-008-clause/",
            "https://choosealicense.com/licenses/bsd-009-clause/",
            "https://choosealicense.com/licenses/bsd-010-clause/",
        ]
        .iter()
        .map(CaseInsensitiveString::new)
        .collect::<Vec<CaseInsensitiveString>>()
    );
}

#[test]
fn test_expand_url_alphabetical_range() {
    let url = "https://choosealicense.com/licenses/[w-z]lib/";

    assert_eq!(
        expand_url(url),
        [
            "https://choosealicense.com/licenses/wlib/",
            "https://choosealicense.com/licenses/xlib/",
            "https://choosealicense.com/licenses/ylib/",
            "https://choosealicense.com/licenses/zlib/",
        ]
        .iter()
        .map(CaseInsensitiveString::new)
        .collect::<Vec<CaseInsensitiveString>>()
    );
}

#[test]
fn test_expand_url_combination() {
    let url = "https://choosealicense.com/licenses/bsd-[2-4]-clause{,-clear}/";

    assert_eq!(
        expand_url(url),
        [
            "https://choosealicense.com/licenses/bsd-2-clause/",
            "https://choosealicense.com/licenses/bsd-2-clause-clear/",
            "https://choosealicense.com/licenses/bsd-3-clause/",
            "https://choosealicense.com/licenses/bsd-3-clause-clear/",
            "https://choosealicense.com/licenses/bsd-4-clause/",
            "https://choosealicense.com/licenses/bsd-4-clause-clear/",
        ]
        .iter()
        .map(CaseInsensitiveString::new)
        .collect::<Vec<CaseInsensitiveString>>()
    );
}

#[test]
fn test_expand_url_empty() {
    let url = "https://choosealicense.com";

    assert_eq!(expand_url(url), Vec::<CaseInsensitiveString>::new());
}

#[test]
fn test_expand_url_percent_encoded() {
    let url = "https://choosealicense.com/licenses/%7Bmit%7D/";

    assert_eq!(expand_url(url), Vec::<CaseInsensitiveString>::new());
}