dom_smoothie 0.18.0

use dom_query::Tree;
use foldhash::{HashMap, HashSet};
use std::vec;

use dom_query::{Document, NodeId, NodeRef};
use flagset::FlagSet;

use crate::config::CandidateSelectMode;
#[allow(clippy::wildcard_imports)]
use crate::glob::*;
use crate::grab_flags::GrabFlags;
#[allow(clippy::wildcard_imports)]
use crate::helpers::*;
#[allow(clippy::wildcard_imports)]
use crate::matching::*;
use crate::prep_article::prep_article;
#[allow(clippy::wildcard_imports)]
use crate::score::*;
use crate::Config;
use crate::Metadata;
use crate::Readability;

impl Readability {
    pub(crate) fn grab_article(&self, metadata: &Metadata) -> Option<Document> {
        let mut flags: FlagSet<GrabFlags> = FlagSet::full();
        let mut best_attempt: Option<(Document, usize)> = None;
        loop {
            let doc = self.doc.clone();
            let article_node = self.attempt_grab_article(&doc, &flags, metadata);
            // Now that we've gone through the full algorithm, check to see if
            // we got any meaningful content. If we didn't, we may need to re-run
            // `grab_article` with different flags set. This gives us a higher likelihood of
            // finding the content, and the sieve approach gives us a higher likelihood of
            // finding the -right- content.

            if let Some(ref article_node) = article_node {
                let text_length = article_node.normalized_char_count();
                if text_length >= self.config.char_threshold {
                    return Some(doc);
                }

                if let Some((_, best_text_length)) = best_attempt {
                    if text_length > best_text_length {
                        best_attempt = Some((doc, text_length));
                    }
                } else {
                    best_attempt = Some((doc, text_length));
                }
            }
            if flags.contains(GrabFlags::StripUnlikelys) {
                flags -= GrabFlags::StripUnlikelys;
            } else if flags.contains(GrabFlags::WeightClasses) {
                flags -= GrabFlags::WeightClasses;
            } else if flags.contains(GrabFlags::CleanConditionally) {
                flags -= GrabFlags::CleanConditionally;
            } else {
                // No luck after removing flags,
                // just return the longest text we found during the different loops
                let (best_doc, _) = best_attempt?;
                return Some(best_doc);
            }
        }
    }

    pub(crate) fn attempt_grab_article<'a>(
        &self,
        doc: &'a Document,
        flags: &FlagSet<GrabFlags>,
        metadata: &Metadata,
    ) -> Option<NodeRef<'a>> {
        let selection = doc.select_single("body");
        let body_node = selection.nodes().first()?;
        let strip_unlikely = flags.contains(GrabFlags::StripUnlikelys);
        let elements_to_score = collect_elements_to_score(body_node, strip_unlikely, metadata);
        let article_node = self.handle_candidates(&elements_to_score, body_node, flags);
        article_node.map(|n| NodeRef::new(n.id, &doc.tree))
    }

    fn handle_candidates<'a>(
        &self,
        elements_to_score: &[NodeRef<'a>],
        body_node: &'a NodeRef,
        flags: &FlagSet<GrabFlags>,
    ) -> Option<NodeRef<'a>> {
        let tree = body_node.tree;
        let weigh_class = flags.contains(GrabFlags::WeightClasses);
        let top_candidates = score_elements(elements_to_score, tree, &self.config, flags);

        let mut top_candidate = top_candidates.first().copied();

        let mut top_candidate_is_created = false;

        if top_candidate.is_none() || top_candidate.as_ref().is_some_and(|n| n.has_name("body")) {
            top_candidate_is_created = true;
            let tc = tree.new_element("div");

            tree.reparent_children_of(&body_node.id, Some(tc.id));
            body_node.append_child(&tc);
            init_node_score(&tc, weigh_class);
            top_candidate = Some(tc);
        } else if let Some(mut tc) = top_candidate {
            if matches!(
                self.config.candidate_select_mode,
                CandidateSelectMode::DomSmoothie
            ) {
                tc = find_common_candidate_alt(tc, &top_candidates, weigh_class);
            } else {
                // Find a better top candidate node if it contains (at least three) nodes which belong to `top_candidates` array
                // and whose scores are quite closed with current `top_candidate` node.
                tc = find_common_candidate(tc, &top_candidates, weigh_class);
            }

            // If the top candidate is the only child, use parent instead. This will help sibling
            // joining logic when adjacent content is actually located in parent's sibling node.
            let mut parent_of_top_candidate = tc.parent();

            while let Some(ref tc_parent) = parent_of_top_candidate {
                if tc_parent.has_name("body") {
                    break;
                }

                if tc_parent.element_children().len() != 1 {
                    break;
                }
                tc = *tc_parent;
                parent_of_top_candidate = tc_parent.parent();
            }
            top_candidate = Some(tc);
        }

        let tc = top_candidate.as_ref()?;

        if !has_node_score(tc) {
            init_node_score(tc, weigh_class);
        }
        // Now that we have the top candidate, look through its siblings for content
        // that might also be related. Things like preambles, content split by ads
        // that we removed, etc.
        let article_content = tree.new_element("div");
        assign_article_node(tc, &article_content);

        //prepare the article
        prep_article(&article_content, flags, &self.config);

        if top_candidate_is_created {
            tc.set_attr("id", CONTENT_ID);
            tc.set_attr("class", "page");
        } else {
            // this code does the same thing as mozilla's implementation, but it is more simpler.
            article_content.set_attr("id", CONTENT_ID);
            article_content.set_attr("class", "page");
        }

        Some(article_content)
    }
}

fn is_unlikely_candidate(node: &NodeRef) -> bool {
    // Assuming that `<body>` node can't can't reach this function
    if node.has_name("a") {
        return false;
    }

    let match_string = get_node_matching_string(node);
    if match_string.is_empty() {
        return false;
    }

    if !match_unlikely(&match_string) {
        return false;
    }

    !has_ancestor(node, Some(0), |n| {
        let Some(qual_name) = n.qual_name_ref() else {
            return false;
        };
        matches!(qual_name.local.as_ref(), "table" | "code")
    })
}

fn div_into_p(node: &NodeRef) {
    // Turn all divs that don't have children block level elements into p's
    // Put phrasing content into paragraphs.
    let mut child_node = node.first_child();
    while let Some(ref child) = child_node {
        child_node = wrap_phrasing_content(child);
    }
}

fn wrap_phrasing_content<'a>(node: &NodeRef<'a>) -> Option<NodeRef<'a>> {
    if is_phrasing_content(node) && !is_whitespace(node) {
        let mut next_sibling = node.next_sibling();
        let p = node.tree.new_element("p");
        node.insert_before(&p);
        p.append_child(node);

        while let Some(child) = next_sibling {
            next_sibling = child.next_sibling();
            if is_phrasing_content(&child) {
                p.append_child(&child);
            } else {
                break;
            }
        }
        // Because `p` starts with phrasing content that is not whitespace,
        // we can skip checking the first child for whitespace.

        while let Some(p_last_child) = p.last_child() {
            if is_whitespace(&p_last_child) {
                p_last_child.remove_from_parent();
            } else {
                break;
            }
        }

        return next_sibling;
    }

    node.next_sibling()
}

fn has_child_block_element(node: &NodeRef) -> bool {
    node.descendants_it().any(|n| {
        n.element_ref()
            .is_some_and(|el| BLOCK_ELEMS.contains(&el.name.local))
    })
}

fn score_elements<'a>(
    elements_to_score: &[NodeRef<'a>],
    tree: &'a Tree,
    cfg: &Config,
    flags: &FlagSet<GrabFlags>,
) -> Vec<NodeRef<'a>> {
    let mut score_map: HashMap<NodeId, f32> = HashMap::default();
    let mut cc_cache = CharCounterCache::default();

    for element in elements_to_score {
        let content_len = cc_cache.char_count(element);
        if content_len < 25 {
            continue;
        }
        // these elements have at least one ancestor -- their parent.
        let ancestors = element.ancestors(Some(5));

        // Count commas in the element's text content without allocating a new StrTendril.
        // Equivalent to `1 + element.text().split(COMMAS).count()`, but more efficient.
        let mut content_score = 2 + score_text_content(element);
        content_score += std::cmp::min(content_len / 100, 3);
        for (level, ancestor) in ancestors.iter().enumerate() {
            if !ancestor.is_element() || ancestor.parent().is_none() {
                continue;
            }

            let score_divider: f32 = match level {
                0 => 1.0,
                1 => 2.0,
                _ => (level * 3) as f32,
            };

            let mut ancestor_score = if let Some(score) = score_map.get(&ancestor.id) {
                *score
            } else {
                score_map.insert(ancestor.id, 0.0);
                determine_node_score(ancestor, flags.contains(GrabFlags::WeightClasses))
            };

            ancestor_score += content_score as f32 / score_divider;
            score_map
                .entry(ancestor.id)
                .and_modify(|s| *s = ancestor_score);

            if ancestor.has_name("body") {
                break;
            }
        }
    }

    // Scale the final candidates score based on link density. Good content
    // should have a relatively small link density (5% or less) and be mostly
    // unaffected by this operation.

    let mut scored_candidates: Vec<_> = score_map
        .into_iter()
        .filter(|(_, score)| *score > 0.0)
        .map(|(node_id, prev_score)| {
            let candidate = NodeRef::new(node_id, tree);
            // Skipping adjustment of low score
            let score = if prev_score > cfg.min_score_to_adjust {
                prev_score * (1.0 - link_density_fn(&candidate, None, |n| cc_cache.char_count(n)))
            } else {
                prev_score
            };
            set_node_score(&candidate, score);
            (candidate, score)
        })
        .collect();

    scored_candidates.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());

    scored_candidates
        .into_iter()
        .take(cfg.n_top_candidates)
        .map(move |c| c.0)
        .collect()
}

fn assign_article_node(tc: &NodeRef, article_content: &NodeRef) {
    let tc_node_score = get_node_score(tc);
    let mut sibling_score_threshold = tc_node_score * 0.2;
    if sibling_score_threshold < 10.0 {
        sibling_score_threshold = 10.0;
    }
    // Keep potential top candidate's parent node to try to get text direction of it later.
    let Some(tc_parent) = tc.parent() else {
        unreachable!("Top candidate must have a parent")
    };

    let tc_class = tc.attr_or("class", "");
    let siblings: Vec<NodeRef> = tc_parent.element_children();
    for sibling in &siblings {
        let mut append = false;
        if sibling.id == tc.id {
            append = true;
        } else {
            let mut content_bonus: f32 = 0.0;
            let sibling_class = sibling.attr_or("class", "");
            if !tc_class.is_empty() && sibling_class == tc_class {
                content_bonus += tc_node_score * 0.2;
            }
            let sibling_score = get_node_score(sibling);
            if sibling_score > 0.0 {
                if sibling_score + content_bonus >= sibling_score_threshold {
                    append = true;
                }
            } else if sibling.has_name("p") {
                let sibling_text = sibling.text();
                let node_content = normalize_spaces(&sibling_text);
                let node_length = node_content.chars().count();
                let link_density = link_density(sibling, Some(node_length));

                if (node_length > 80 && link_density < 0.25)
                    || node_length < 80
                        && node_length > 0
                        && link_density == 0.0
                        && is_sentence(&node_content)
                {
                    append = true;
                }
            }
        }

        //appending sibling
        if append {
            if !node_name_in(sibling, &ALTER_TO_DIV_EXCEPTIONS) {
                // We have a node that isn't a common block level element, like a form or td tag.
                // Turn it into a div so it doesn't get filtered out later by accident.
                sibling.rename("div");
            }
            article_content.append_child(&sibling.id);
        }
    }
    tc_parent.append_child(article_content);
}

/// Find a better top candidate across other candidates in a way that `mozilla/readability` does.
fn find_common_candidate<'a>(
    mut top_candidate: NodeRef<'a>,
    top_candidates: &[NodeRef<'a>],
    weigh_class: bool,
) -> NodeRef<'a> {
    let tc = &mut top_candidate;
    let tc_score = get_node_score(tc);

    let mut alternative_candidate_ancestors = vec![];
    for alt in top_candidates.iter().skip(1) {
        if get_node_score(alt) / tc_score >= 0.75 {
            alternative_candidate_ancestors.push(alt.ancestors(Some(0)));
        }
    }
    // MIN_COMMON_ANCESTORS (in mozilla/readability.js -- MINIMUM_TOPCANDIDATES)
    // represents the number of top candidates' ancestors that may be common.
    // The idea is good, but this magic number doesn't always work very well.
    // For example, imagine we have only two candidates, and both are significant.
    // So, we end up with one top candidate and another candidate.
    // However, the second candidate will be excluded in the end because we require
    // at least three (!) lists of ancestors,
    // which is impossible to derive from just one candidate.
    // To adjust the top candidate to share a common ancestor with other candidates,
    // we would need at least three other candidates.
    // Currently, I consider this approach to be flawed...

    if alternative_candidate_ancestors.len() > MIN_COMMON_ANCESTORS {
        let mut parent_of_top_candidate = tc.parent();
        while let Some(ref tc_parent) = parent_of_top_candidate {
            if tc_parent.has_name("body") {
                break;
            }

            let mut lists_containing_this_ancestor = 0;

            for alt_ancestor in &alternative_candidate_ancestors {
                if alt_ancestor.iter().any(|n| n.id == tc_parent.id) {
                    lists_containing_this_ancestor += 1;
                }
            }

            if lists_containing_this_ancestor >= MIN_COMMON_ANCESTORS {
                top_candidate = *tc_parent;
                break;
            }

            parent_of_top_candidate = tc_parent.parent();
        }
    }

    top_candidate = adjust_top_candidate_by_parent(top_candidate, weigh_class);

    top_candidate
}

/// Find a better top candidate across other candidates (alternative approach).
fn find_common_candidate_alt<'a>(
    mut top_candidate: NodeRef<'a>,
    top_candidates: &[NodeRef<'a>],
    weigh_class: bool,
) -> NodeRef<'a> {
    if top_candidates.len() < 2 {
        return top_candidate;
    }
    let tc = &mut top_candidate;

    let tc_ancestors = get_node_ancestors_set(tc);
    let tc_score = get_node_score(tc);

    let mut ancestor_match_counter: HashMap<NodeId, usize> = HashMap::default();

    for alt in top_candidates.iter().skip(1) {
        if get_node_score(alt) / tc_score >= 0.75 {
            let alt_ancestors = get_node_ancestors_set(alt);
            if alt_ancestors.contains(&tc.id) {
                continue;
            }
            let intersect = tc_ancestors.intersection(&alt_ancestors);
            for item in intersect {
                *ancestor_match_counter.entry(*item).or_insert(0) += 1;
            }
        }
    }

    let mut require_adjustment = true;
    // choosing the best candidate by how close it to the top candidate,
    // and then by how many common ancestors it has across all other candidates
    if let Some(best_candidate_id) = ancestor_match_counter
        .into_iter()
        .max_by(|x, y| x.0.cmp(&y.0).then(x.1.cmp(&y.1)))
        .map(|n| n.0)
    {
        let best_candidate = NodeRef::new(best_candidate_id, tc.tree);
        if get_node_score(&best_candidate) > tc_score / 3.0 {
            top_candidate = best_candidate;
            require_adjustment = false;
        }
    }

    if require_adjustment {
        top_candidate = adjust_top_candidate_by_parent(top_candidate, weigh_class);
    }
    top_candidate
}

fn get_node_ancestors_set(node: &NodeRef) -> HashSet<NodeId> {
    // only elements, no html or body, and have a score
    node.ancestors(Some(0))
        .iter()
        .filter(|n| {
            n.is_element()
                && !matches!(n.node_name().as_deref(), Some("html" | "body"))
                && has_node_score(n)
        })
        .map(|n| n.id)
        .collect::<HashSet<_>>()
}

fn adjust_top_candidate_by_parent(
    mut top_candidate: NodeRef<'_>,
    weigh_class: bool,
) -> NodeRef<'_> {
    let tc = &mut top_candidate;
    if !has_node_score(tc) {
        init_node_score(tc, weigh_class);
    }
    // Because of our bonus system, parents of candidates might have scores
    // themselves. They get half of the node. There won't be nodes with higher
    // scores than our `top_candidate`, but if we see the score going *up* in the first
    // few steps up the tree, that's a decent sign that there might be more content
    // lurking in other places that we want to unify in. The sibling stuff
    // below does some of that - but only if we've looked high enough up the DOM
    // tree.
    let mut last_score = get_node_score(tc);
    let score_threshold = last_score / 3.0;
    let mut parent_of_top_candidate = tc.parent();
    while let Some(ref tc_parent) = parent_of_top_candidate {
        if tc_parent.has_name("body") {
            break;
        }

        if !has_node_score(tc_parent) {
            parent_of_top_candidate = tc_parent.parent();
            continue;
        }

        let parent_score = get_node_score(tc_parent);
        if parent_score < score_threshold {
            break;
        }
        if parent_score > last_score {
            top_candidate = *tc_parent;
            break;
        }
        last_score = parent_score;
        parent_of_top_candidate = tc_parent.parent();
    }
    top_candidate
}

/// Collecting nodes to score. Also, it removes unlikely candidates and elements without content.
fn collect_elements_to_score<'a>(
    root_node: &'a NodeRef,
    strip_unlikely: bool,
    metadata: &Metadata,
) -> Vec<NodeRef<'a>> {
    let tree = &root_node.tree;
    let mut elements_id_to_score: Vec<NodeId> = vec![];
    let mut should_remove_title_header = !metadata.title.is_empty();
    let mut next_node = next_child_or_sibling(root_node, false);
    while let Some(mut node) = next_node {
        if !is_probably_visible(&node) {
            next_node = next_child_or_sibling(&node, true);
            node.remove_from_parent();
            continue;
        }

        if node.has_name("svg") {
            next_node = next_child_or_sibling(&node, true);
            continue;
        }

        if MATCHER_DIALOGS.match_element(&node) {
            next_node = next_child_or_sibling(&node, true);
            node.remove_from_parent();
            continue;
        }

        if should_remove_title_header
            && MATCHER_HEADING.match_element(&node)
            && text_similarity(&metadata.title, &node.text()) > 0.75
        {
            should_remove_title_header = false;
            next_node = next_child_or_sibling(&node, true);
            node.remove_from_parent();
            continue;
        }

        if strip_unlikely {
            let strip = is_unlikely_candidate(&node)
                || node
                    .attr("role")
                    .is_some_and(|role| UNLIKELY_ROLES.contains(&role));
            if strip {
                next_node = next_child_or_sibling(&node, true);
                node.remove_from_parent();
                continue;
            }
        }
        if node_name_in(&node, &TAGS_WITH_CONTENT) && is_element_without_content(&node) {
            next_node = next_child_or_sibling(&node, true);
            node.remove_from_parent();
            continue;
        }

        if node_name_in(&node, &DEFAULT_TAGS_TO_SCORE) {
            elements_id_to_score.push(node.id);
        }

        if node.has_name("div") {
            div_into_p(&node);

            // Sites like http://mobile.slate.com encloses each paragraph with a DIV
            // element. DIVs with only a P element inside and no text content can be
            // safely converted into plain P elements to avoid confusing the scoring
            // algorithm with DIVs with are, in practice, paragraphs.

            // Check `p` first (cheap), then link density (expensive).
            let single_p: Option<NodeRef<'_>> =
                single_child_element(&node, "p").filter(|_| link_density(&node, None) < 0.25);
            if let Some(new_node) = single_p {
                node.replace_with(&new_node);
                elements_id_to_score.push(new_node.id);
                node = new_node;
            } else if !has_child_block_element(&node) {
                node.rename("p");
                elements_id_to_score.push(node.id);
            }
        }
        next_node = next_child_or_sibling(&node, false);
    }
    elements_id_to_score
        .iter()
        .map(|n| NodeRef::new(*n, tree))
        .collect()
}

#[cfg(not(feature = "aho-corasick"))]
fn match_unlikely(haystack: &str) -> bool {
    let check = BytePatternCheck::new(haystack);

    if !check.contains_any(UNLIKELY_CANDIDATES) {
        return false;
    }
    if check.contains_any(MAYBE_CANDIDATES) {
        return false;
    }
    true
}

#[cfg(feature = "aho-corasick")]
fn match_unlikely(haystack: &str) -> bool {
    if !crate::ac_automat::AC_UNLIKELY.is_match(haystack) {
        return false;
    }
    if crate::ac_automat::AC_MAYBE.is_match(haystack) {
        return false;
    }
    true
}

#[cfg(test)]
mod tests {

    use super::*;
    use crate::readability::Readability;

    #[test]
    fn test_removing_probably_invisible_nodes() {
        let contents = r#"<!DOCTYPE>
        <html>
            <head><title>Test</title></head>
            <body>
                 <p hidden>This paragraph should be hidden.</p>
                 <p aria-hidden="true">This paragraph should be hidden.</p>
                 <p style="display:none">This paragraph should be hidden.</p>
                 <p style="display: none !important">This paragraph should be hidden.</p>
                 <p style="display: none!important">This paragraph should be visible.</p>
                 <p style="display:">This paragraph should be visible.</p>
                 <p style="display">This paragraph should be visible.</p>
                 <p style=":">This paragraph should be visible.</p>
                 <p style="visibility:hidden">This paragraph should be hidden.</p>
                 <p aria-hidden="true" class="mwe-math-fallback-image-inline">123*123</p>
                 <p>This paragraph is visible</p>
                 <p style="DISPLAY: NONE">This paragraph should be hidden.</p>
                 <p style="display: none; visibility: visible">This paragraph should be hidden.</p>
                 <p style="font-family: 'Times New Roman'; display: none">This paragraph should be hidden.</p>
            </body>
        </html>"#;

        let doc = Document::from(contents);
        let body = doc.body().unwrap();
        collect_elements_to_score(&body, true, &Metadata::default());
        assert_eq!(6, doc.select("p").length());
    }

    #[test]
    fn test_remove_dialog() {
        let contents = r#"<!DOCTYPE>
        <html>
            <head><title>Test</title></head>
            <body>
                 <div id="dialog1" role="dialog" aria-modal="true">
                    <h2>Test dialog<h2>
                    <button id="close1">Close</button>
                 </div>
            </body>
        </html>"#;

        let doc = Document::from(contents);
        assert!(doc.select("#dialog1").exists());
        let body = doc.body().unwrap();
        collect_elements_to_score(&body, true, &Metadata::default());

        assert!(!doc.select("#dialog1").exists());
        assert!(!doc.select("#close1").exists());
    }

    #[test]
    fn test_unlikely_roles() {
        let contents = r#"<!DOCTYPE>
        <html>
            <head><title>Test</title></head>
            <body>
                 <div id="dialog1" role="dialog">
                    <h2>Test dialog<h2>
                    <button id="close1">Close</button>
                 </div>
                 <nav id="nav1" role="navigation"></nav>
            </body>
        </html>"#;

        let doc = Document::from(contents);
        assert!(doc.select("*[role]").exists());
        let body = doc.body().unwrap();

        collect_elements_to_score(&body, true, &Metadata::default());
        assert!(!doc.select("*[role]").exists());
    }

    #[test]
    fn test_remove_empty() {
        let contents = r"<!DOCTYPE>
        <html>
            <head><title>Test</title></head>
            <body>
                 <p>This paragraph is visible</p>
                 <header></header>
                 <section></section>
                 <div></div>
                 <h1></h1>
                 <h2></h2>
                 <h3></h3>
                 <h4></h4>
                 <h5></h5>
                 <h6></h6>
            </body>
        </html>";

        let ra = Readability::new(contents, None, None).unwrap();
        let sel = ra.doc.select("body > *");
        let count_before = sel.nodes().iter().filter(|n| n.is_element()).count();

        assert_eq!(count_before, 10);

        let clean_doc = ra.grab_article(&Metadata::default()).unwrap();
        let sel = clean_doc.select("body > *");
        let count_after = sel.nodes().iter().filter(|n| n.is_element()).count();
        assert_eq!(count_after, 1);
    }

    #[test]
    fn test_remove_title_duplicates() {
        let contents = r"<!DOCTYPE>
        <html>
            <head><title>Rust (programming language) - Wikipedia</title></head>
            <body>
                 <h1>Rust (programming language)</h1>
            </body>
        </html>";

        let readability = Readability::from(contents);
        let metadata = readability.get_article_metadata(None);
        let body = readability.doc.body().unwrap();

        assert!(readability.doc.select("h1").exists());
        collect_elements_to_score(&body, true, &metadata);
        assert!(!readability.doc.select("h1").exists());
    }

    #[test]
    fn test_remove_unlikely_candidates() {
        let contents = r#"<!DOCTYPE>
        <html>
            <head><title>Test</title></head>
            <body>
                 <h1>Test</h1>
                 <div class="banner">Some annoying content</div>
            </body>
        </html>"#;

        let doc = Document::from(contents);
        assert!(doc.select("div.banner").exists());
        let body = doc.body().unwrap();
        collect_elements_to_score(&body, true, &Metadata::default());
        assert!(!doc.select("div.banner").exists());
    }
    #[test]
    fn test_skip_ok_maybe_candidates() {
        let contents = r#"<!DOCTYPE>
        <html>
            <head><title>Test</title></head>
            <body>
                 <h1>Test</h1>
                 <a class="banner">Some annoying content</a>
            </body>
        </html>"#;

        let doc = Document::from(contents);
        assert!(doc.select("a.banner").exists());
        let body = doc.body().unwrap();
        collect_elements_to_score(&body, true, &Metadata::default());
        assert!(doc.select("a.banner").exists());
    }
}