jscpd-rs 0.1.6

use super::*;
use crate::cli::{Mode, Options};

fn token_slices<'a>(content: &'a str, tokens: &[DetectionToken]) -> Vec<&'a str> {
    tokens
        .iter()
        .map(|token| &content[token.range[0]..token.range[1]])
        .collect()
}

#[test]
fn tokenizes_non_whitespace_tokens_with_locations() {
    let tokens =
        tokenize_for_detection("let a = 1;\nlet b = 2;", "javascript", &Options::default());
    assert_eq!(tokens[0].start.line, 1);
    assert_eq!(tokens[5].start.line, 2);
}

#[test]
fn public_tokenizer_generates_source_maps_like_upstream_entrypoint() {
    let tokenizer = Tokenizer::new();

    let maps = tokenizer.generate_maps("snippet.js", "const a = 1;\nconst b = 2;", "javascript");

    assert_eq!(maps.len(), 1);
    assert_eq!(maps[0].source_id, "snippet.js");
    assert_eq!(maps[0].format, "javascript");
    assert_eq!(maps[0].tokens.len(), 10);
    assert_eq!(maps[0].lines, 1);
    assert_eq!(maps[0].tokens[0].start.line, 1);
}

#[test]
fn public_tokenizer_uses_configured_options() {
    let tokenizer = Tokenizer::with_options(Options {
        mode: Mode::Weak,
        ..Options::default()
    });

    let tokens = tokenizer.tokenize("const a = 1; // comment\n", "javascript");

    assert_eq!(tokens.len(), 5);
}

#[test]
fn skips_ignore_regions() {
    let content = "keep\n// jscpd:ignore-start\nskip\n// jscpd:ignore-end\nkeep2\n";
    let tokens = tokenize_for_detection(content, "javascript", &Options::default());
    assert_eq!(tokens.len(), 2);
}

#[test]
fn detection_tokenizer_avoids_token_value_allocations() {
    let tokens =
        tokenize_for_detection("let a = 1;\nlet b = 2;", "javascript", &Options::default());
    assert_eq!(tokens.len(), 10);
    assert_eq!(tokens[0].start.line, 1);
    assert_eq!(tokens[5].start.line, 2);
}

#[test]
fn js_like_json_report_positions_count_prism_whitespace_tokens() {
    let options = Options {
        reporters: vec!["json".to_string()],
        ..Options::default()
    };
    for format in ["javascript", "typescript", "jsx", "tsx"] {
        let tokens = tokenize_for_detection("let a = 1;\nlet b = 2;", format, &options);
        assert_eq!(tokens[0].start.position, 0);
        assert_eq!(tokens[1].start.position, 2);
        assert_eq!(tokens[5].start.position, 9);
    }
}

#[test]
fn jsx_attribute_expression_emits_embedded_javascript_map() {
    let maps = tokenize_maps_for_detection(
        "const x = <div className={classNames(className, classes)} />;",
        "jsx",
        &Options::default(),
    );
    assert_eq!(maps.len(), 2);
    assert_eq!(maps[0].format, "jsx");
    assert_eq!(maps[1].format, "javascript");

    let embedded = &maps[1].tokens;
    assert_eq!(embedded.len(), 9);
    assert_eq!(
        embedded.last().unwrap().end.position - embedded.first().unwrap().start.position,
        8
    );
}

#[test]
fn jsx_embedded_javascript_keeps_nested_object_whitespace() {
    let content = "const x = <A p={{\n  color: PRIMARY_COLOR\n}} />;";
    let maps = tokenize_maps_for_detection(content, "tsx", &Options::default());
    let embedded = maps
        .iter()
        .find(|map| map.format == "javascript")
        .expect("embedded javascript map");

    assert!(
        embedded
            .tokens
            .iter()
            .any(|token| &content[token.range[0]..token.range[1]] == "\n  ")
    );
}

#[test]
fn jsx_text_is_split_like_javascript_text() {
    let content = r#"const x = <div>Hello, "Go" this.</div>;"#;
    let tokens = tokenize_for_detection(content, "javascript", &Options::default());
    let values = token_slices(content, &tokens);

    assert!(values.contains(&"Hello"));
    assert!(values.contains(&","));
    assert!(values.contains(&r#""Go""#));
    assert!(values.contains(&"this"));
}

#[test]
fn jsx_text_unclosed_quote_stops_at_line_end() {
    let content = "const x = <div>\"Captured an\nerror: null\". Clicking</div>;";
    let tokens = tokenize_for_detection(content, "javascript", &Options::default());
    let values = token_slices(content, &tokens);

    assert!(values.contains(&"\"Captured"));
    assert!(values.contains(&"error"));
    assert!(values.contains(&"null"));
    assert!(values.contains(&"\""));
    assert!(values.contains(&"."));
}

#[test]
fn jsx_dashed_identifiers_are_split_like_prism() {
    let content = r#"expect(root).toMatchRenderedOutput(<suspensey-thing src="A" />);"#;
    let tokens = tokenize_for_detection(content, "javascript", &Options::default());
    let values = token_slices(content, &tokens);

    assert!(values.contains(&"suspensey"));
    assert!(values.contains(&"-"));
    assert!(values.contains(&"thing"));
    assert!(!values.contains(&"suspensey-thing"));
}

#[test]
fn js_spread_token_is_operator_like_prism() {
    let content = "const next = [...items];";
    let tokens = tokenize_for_detection(content, "javascript", &Options::default());
    let spread = tokens
        .iter()
        .find(|token| &content[token.range[0]..token.range[1]] == "...")
        .expect("spread token");

    assert_eq!(spread.hash, hash_token(TokenKind::Operator, "...", false));
}

#[test]
fn generic_tokenizer_handles_common_non_native_formats() {
    for format in ["css", "markup", "yaml", "toml", "python"] {
        let maps = tokenize_maps_for_detection("alpha beta\n  gamma", format, &Options::default());

        assert_eq!(maps.len(), 1);
        assert_eq!(maps[0].format, format);
        assert_eq!(maps[0].tokens.len(), 3);
    }
}

#[test]
fn all_supported_formats_have_a_tokenizer_smoke_path() {
    for format in crate::formats::supported_formats() {
        let content = smoke_content_for_format(format);
        let maps = tokenize_maps_for_detection(content, format, &Options::default());
        assert!(
            maps.iter().any(|map| !map.tokens.is_empty()),
            "format {format} produced no tokens"
        );
        assert!(
            maps.iter()
                .all(|map| crate::formats::supported_formats().contains(&map.format.as_str())),
            "format {format} produced an unsupported embedded map"
        );
    }
}

fn smoke_content_for_format(format: &str) -> &'static str {
    match format {
        "astro" => "---\nconst title = 'Demo';\n---\n<section>{title}</section>\n",
        "jsx" => "const view = <section>{title}</section>;\n",
        "markdown" => "# Demo\n\n```js\nconst value = 1;\n```\n",
        "markup" => "<section><span>alpha beta</span></section>\n",
        "svelte" => "<script>let title = 'Demo';</script>\n<h1>{title}</h1>\n",
        "tsx" => "const view: JSX.Element = <section>{title}</section>;\n",
        "vue" => "<template>\n  <section>{{ title }}</section>\n</template>\n",
        _ => "alpha beta gamma\nalpha beta delta\n",
    }
}

#[test]
fn haml_comment_block_is_single_comment_token() {
    let content = "%section\n  %p Same\n-# File-specific comment\n  .settings\n    %h2 Title\n";
    let tokens = tokenize_for_detection(content, "haml", &Options::default());
    let comment = tokens
        .iter()
        .find(|token| content[token.range[0]..token.range[1]].starts_with("-#"))
        .expect("haml comment token");

    assert_eq!(comment.start.line, 3);
    assert_eq!(comment.end.line, 5);
    assert_eq!(
        &content[comment.range[0]..comment.range[1]],
        "-# File-specific comment\n  .settings\n    %h2 Title"
    );
}

#[test]
fn pug_dot_block_is_single_plain_text_token() {
    let content = "style.\n  .panel {\n    color: red;\n  }\nbody\n";
    let tokens = tokenize_for_detection(content, "pug", &Options::default());
    let block = tokens
        .iter()
        .find(|token| content[token.range[0]..token.range[1]].starts_with("style."))
        .expect("pug dot block token");

    assert_eq!(block.start.line, 1);
    assert_eq!(block.end.line, 4);
    assert_eq!(
        &content[block.range[0]..block.range[1]],
        "style.\n  .panel {\n    color: red;\n  }"
    );
}

#[test]
fn markdown_fenced_javascript_emits_embedded_map() {
    let content = "# Demo\n\n```js\nfunction alpha() {\n  return 42;\n}\n```\n";
    let maps = tokenize_maps_for_detection(content, "markdown", &Options::default());

    assert!(maps.iter().any(|map| map.format == "markdown"));
    let javascript = maps
        .iter()
        .find(|map| map.format == "javascript")
        .expect("embedded javascript map");

    assert_eq!(javascript.tokens[0].start.line, 4);
    assert_eq!(javascript.tokens[0].start.column, 1);
    assert_eq!(
        &content[javascript.tokens[0].range[0]..javascript.tokens[0].range[1]],
        "function"
    );
}

#[test]
fn markdown_fenced_code_is_removed_from_markdown_map() {
    let content = "before\n\n```ts\nconst hidden = true;\n```\n\nafter\n";
    let maps = tokenize_maps_for_detection(content, "markdown", &Options::default());
    let markdown = maps
        .iter()
        .find(|map| map.format == "markdown")
        .expect("markdown map");
    let markdown_values = token_slices(content, &markdown.tokens);

    assert!(markdown_values.contains(&"before"));
    assert!(markdown_values.contains(&"after"));
    assert!(!markdown_values.contains(&"hidden"));
}

#[test]
fn markdown_fenced_gap_tokens_stay_on_their_lines() {
    let content = "before\n```ts\nconst hidden = true;\n```\nafter\n";
    let maps = tokenize_maps_for_detection(content, "markdown", &Options::default());
    let markdown = maps
        .iter()
        .find(|map| map.format == "markdown")
        .expect("markdown map");
    let gap_tokens = markdown
        .tokens
        .iter()
        .filter(|token| (2..=4).contains(&token.start.line))
        .collect::<Vec<_>>();

    assert!(gap_tokens.iter().any(|token| token.start.line == 3));
    assert!(
        gap_tokens
            .iter()
            .all(|token| token.start.line == token.end.line)
    );
}

#[test]
fn markdown_fenced_typescript_uses_language_name() {
    let content = "```typescript\ntype Answer = number;\n```\n";
    let maps = tokenize_maps_for_detection(content, "markdown", &Options::default());

    assert!(maps.iter().any(|map| map.format == "typescript"));
}

#[test]
fn markdown_front_matter_emits_yaml_map() {
    let content = "---\ntitle: Demo\ntags:\n  - docs\n---\n# Demo\n";
    let maps = tokenize_maps_for_detection(content, "markdown", &Options::default());
    let yaml = maps
        .iter()
        .find(|map| map.format == "yaml")
        .expect("front matter yaml map");

    assert_eq!(yaml.tokens[0].start.line, 2);
    assert_eq!(
        &content[yaml.tokens[0].range[0]..yaml.tokens[0].range[1]],
        "title"
    );
    assert_eq!(
        &content[yaml.tokens[1].range[0]..yaml.tokens[1].range[1]],
        ":"
    );
}

#[test]
fn markdown_embedded_generic_blocks_keep_whitespace_tokens() {
    let content =
        "```coffeescript\njscpd = require 'jscpd'\nresult = jscpd::run\n  reporter: json\n```\n";
    let maps = tokenize_maps_for_detection(content, "markdown", &Options::default());
    let coffeescript = maps
        .iter()
        .find(|map| map.format == "coffeescript")
        .expect("coffeescript map");

    assert!(
        coffeescript
            .tokens
            .iter()
            .any(|token| &content[token.range[0]..token.range[1]] == "\n")
    );
}

#[test]
fn markup_emits_embedded_script_and_style_maps() {
    let content = "<html>\n<script language=\"JavaScript\">\nfunction demo() { return 1; }\n</script>\n<style type=\"text/css\">\nbody { color: red; }\n</style>\n</html>\n";
    let maps = tokenize_maps_for_detection(content, "markup", &Options::default());

    assert!(maps.iter().any(|map| map.format == "markup"));
    let javascript = maps
        .iter()
        .find(|map| map.format == "javascript")
        .expect("embedded javascript map");
    let css = maps
        .iter()
        .find(|map| map.format == "css")
        .expect("embedded css map");

    assert_eq!(javascript.tokens[0].start.line, 3);
    assert_eq!(
        &content[javascript.tokens[0].range[0]..javascript.tokens[0].range[1]],
        "function"
    );
    let body = css
        .tokens
        .iter()
        .find(|token| &content[token.range[0]..token.range[1]] == "body")
        .expect("body selector token");
    assert_eq!(body.start.line, 6);
}

#[test]
fn markup_emits_inline_style_attr_css_map() {
    let content = "<h4  style=\"visibility: hidden\">Order Search</h4>\n";
    let maps = tokenize_maps_for_detection(content, "markup", &Options::default());

    let css = maps
        .iter()
        .find(|map| map.format == "css")
        .expect("inline style css map");
    let values = token_slices(content, &css.tokens);

    assert_eq!(
        values,
        vec!["  ", "style", "=\"", "visibility", ":", " hidden", "\""]
    );
    assert_eq!(css.tokens[0].start.line, 1);
    assert_eq!(css.tokens[0].start.column, 4);

    let markup = maps
        .iter()
        .find(|map| map.format == "markup")
        .expect("markup map");
    assert!(
        !markup
            .tokens
            .iter()
            .any(|token| &content[token.range[0]..token.range[1]] == "style")
    );
}

#[test]
fn markup_inline_style_attr_respects_ignore_regions() {
    let content = "<!-- jscpd:ignore-start -->\n<h4 style=\"visibility: hidden\">Order Search</h4>\n<!-- jscpd:ignore-end -->\n";
    let maps = tokenize_maps_for_detection(content, "markup", &Options::default());

    assert!(maps.iter().all(|map| map.format != "css"));
}

#[test]
fn vue_sfc_emits_template_script_and_style_maps() {
    let content = "<template>\n  <section>{{ title }}</section>\n</template>\n<style lang=\"scss\">\n.panel { color: red; }\n</style>\n<script setup lang=\"ts\">\nconst title: string = 'Demo';\n</script>\n";
    let maps = tokenize_maps_for_detection(content, "vue", &Options::default());

    assert!(maps.iter().any(|map| map.format == "markup"));
    assert!(maps.iter().any(|map| map.format == "scss"));
    let typescript = maps
        .iter()
        .find(|map| map.format == "typescript")
        .expect("typescript map");

    assert_eq!(typescript.tokens[0].start.line, 8);
    assert_eq!(
        &content[typescript.tokens[0].range[0]..typescript.tokens[0].range[1]],
        "const"
    );
}

#[test]
fn vue_sfc_trims_edge_whitespace_from_embedded_block_maps() {
    let content = "<template>\n  <section>{{ title }}</section>\n</template>\n<style lang=\"scss\">\n.panel { color: red; }\n</style>\n";
    let maps = tokenize_maps_for_detection(content, "vue", &Options::default());
    let markup = maps
        .iter()
        .find(|map| map.format == "markup")
        .expect("markup map");
    let scss = maps
        .iter()
        .find(|map| map.format == "scss")
        .expect("scss map");

    assert_eq!(markup.tokens[0].start.line, 2);
    assert_eq!(markup.tokens.last().unwrap().end.line, 2);
    assert_eq!(scss.tokens[0].start.line, 5);
    assert_eq!(scss.tokens.last().unwrap().end.line, 5);
}

#[test]
fn vue_sfc_style_blocks_skip_internal_whitespace_tokens() {
    let content = "<style lang=\"scss\">\n.panel {\n  color: red;\n}\n</style>\n";
    let maps = tokenize_maps_for_detection(content, "vue", &Options::default());
    let scss = maps
        .iter()
        .find(|map| map.format == "scss")
        .expect("scss map");
    let values = token_slices(content, &scss.tokens);

    assert!(
        !values
            .iter()
            .any(|value| value.chars().all(char::is_whitespace))
    );
}

#[test]
fn vue_template_emits_inline_style_attr_css_map() {
    let content = "<template>\n  <div style=\"color: red\">{{ title }}</div>\n</template>\n";
    let maps = tokenize_maps_for_detection(content, "vue", &Options::default());

    let css = maps
        .iter()
        .find(|map| map.format == "css")
        .expect("inline style css map");
    let values = token_slices(content, &css.tokens);

    assert_eq!(
        values,
        vec![" ", "style", "=\"", "color", ":", " red", "\""]
    );
    assert_eq!(css.tokens[0].start.line, 2);
    assert_eq!(css.tokens[0].start.column, 7);
}

#[test]
fn svelte_sfc_emits_markup_script_and_style_maps() {
    let content = "<script>\nlet title = 'Demo';\n</script>\n<h1>{title}</h1>\n<style>\nh1 { color: red; }\n</style>\n";
    let maps = tokenize_maps_for_detection(content, "svelte", &Options::default());

    assert!(maps.iter().any(|map| map.format == "markup"));
    assert!(maps.iter().any(|map| map.format == "javascript"));
    let css = maps
        .iter()
        .find(|map| map.format == "css")
        .expect("css map");
    let h1 = css
        .tokens
        .iter()
        .find(|token| &content[token.range[0]..token.range[1]] == "h1")
        .expect("h1 selector token");

    assert_eq!(h1.start.line, 6);
}

#[test]
fn astro_sfc_emits_frontmatter_script_style_and_markup_maps() {
    let content = "---\nconst title: string = 'Demo';\n---\n<article>{title}</article>\n<script>\nconsole.log(title);\n</script>\n<style>\narticle { color: red; }\n</style>\n";
    let maps = tokenize_maps_for_detection(content, "astro", &Options::default());

    assert!(maps.iter().any(|map| map.format == "markup"));
    assert!(maps.iter().any(|map| map.format == "javascript"));
    assert!(maps.iter().any(|map| map.format == "css"));
    let typescript = maps
        .iter()
        .find(|map| map.format == "typescript")
        .expect("frontmatter typescript map");

    assert_eq!(typescript.tokens[0].start.line, 2);
    assert_eq!(
        &content[typescript.tokens[0].range[0]..typescript.tokens[0].range[1]],
        "const"
    );
}

#[test]
fn astro_markup_trims_blanked_frontmatter_whitespace() {
    let content = "---\nconst title = 'Hello';\n---\n\n<main>{title}</main>\n";
    let maps = tokenize_maps_for_detection(content, "astro", &Options::default());
    let markup = maps
        .iter()
        .find(|map| map.format == "markup")
        .expect("markup map");

    assert_eq!(markup.tokens[0].start.line, 5);
    assert_eq!(
        &content[markup.tokens[0].range[0]..markup.tokens[0].range[1]],
        "<"
    );
}

#[test]
fn apex_soql_blocks_emit_sql_map() {
    let content = "public class A {\n  Account acc = [\n    SELECT Id\n    FROM Account\n  ];\n}\n";
    let maps = tokenize_maps_for_detection(content, "apex", &Options::default());

    assert!(maps.iter().any(|map| map.format == "apex"));
    let sql = maps
        .iter()
        .find(|map| map.format == "sql")
        .expect("sql map");
    let first = sql.tokens.first().expect("sql token");

    assert_eq!(first.start.line, 2);
    assert_eq!(&content[first.range[0]..first.range[1]], "[");
}

#[test]
fn tap_yamlish_blocks_emit_yaml_map() {
    let content = "not ok 1 - failed\n  ---\n  message: Expected value\n  actual: null\n  ...\n";
    let maps = tokenize_maps_for_detection(content, "tap", &Options::default());
    let yaml = maps
        .iter()
        .find(|map| map.format == "yaml")
        .expect("yaml map");

    assert_eq!(yaml.tokens[0].start.line, 2);
    assert_eq!(yaml.tokens[0].start.column, 3);
    assert_eq!(
        &content[yaml.tokens[0].range[0]..yaml.tokens[0].range[1]],
        "---"
    );
    assert!(maps.iter().any(|map| map.format == "tap"));
}

#[test]
fn weak_mode_skips_generic_comments() {
    let content = "# first comment\nalpha beta\n// second comment\ngamma\n";
    let weak_options = Options {
        mode: crate::cli::Mode::Weak,
        ..Options::default()
    };

    let strong = tokenize_for_detection(content, "yaml", &Options::default());
    let weak = tokenize_for_detection(content, "yaml", &weak_options);

    assert_eq!(strong.len(), 5);
    assert_eq!(weak.len(), 3);
}

#[test]
fn yaml_quoted_scalars_are_single_string_tokens() {
    let content = "email: \"jane@example.com\"\n";
    let tokens = tokenize_for_detection(content, "yaml", &Options::default());
    let values = token_slices(content, &tokens);

    assert_eq!(values, vec!["email", ":", "\"jane@example.com\""]);
}

#[test]
fn strict_mode_keeps_generic_whitespace_tokens() {
    let content = "alpha beta\ngamma";
    let strict_options = Options {
        mode: crate::cli::Mode::Strict,
        ..Options::default()
    };

    let mild = tokenize_for_detection(content, "yaml", &Options::default());
    let strict = tokenize_for_detection(content, "yaml", &strict_options);
    let token_values = token_slices(content, &strict);

    assert_eq!(mild.len(), 3);
    assert_eq!(token_values, vec!["alpha", " ", "beta", "\n", "gamma"]);
}

#[test]
fn strict_mode_keeps_js_whitespace_tokens() {
    let content = "let a = 1;\nlet b = 2;";
    let strict_options = Options {
        mode: crate::cli::Mode::Strict,
        ..Options::default()
    };

    let mild = tokenize_for_detection(content, "javascript", &Options::default());
    let strict = tokenize_for_detection(content, "javascript", &strict_options);
    let token_values = token_slices(content, &strict);

    assert_eq!(mild.len(), 10);
    assert!(strict.len() > mild.len());
    assert!(token_values.contains(&" "));
    assert!(token_values.contains(&"\n"));
}

#[test]
fn weak_mode_skips_generic_double_dash_comments() {
    let content = "-- first comment\nselect one\n-- second comment\nfrom table\n";
    let weak_options = Options {
        mode: crate::cli::Mode::Weak,
        ..Options::default()
    };

    let strong = tokenize_for_detection(content, "sql", &Options::default());
    let weak = tokenize_for_detection(content, "sql", &weak_options);
    let token_values = token_slices(content, &weak);

    assert_eq!(strong.len(), 6);
    assert_eq!(token_values, vec!["select", "one", "from", "table"]);
}

#[test]
fn weak_mode_skips_generic_semicolon_comments() {
    let content = "; first comment\n[main]\nkey=value\n  ; second comment\nother=value\n";
    let weak_options = Options {
        mode: crate::cli::Mode::Weak,
        ..Options::default()
    };

    let strong = tokenize_for_detection(content, "ini", &Options::default());
    let weak = tokenize_for_detection(content, "ini", &weak_options);
    let token_values = token_slices(content, &weak);

    assert_eq!(strong.len(), 11);
    assert_eq!(
        token_values,
        vec!["[", "main", "]", "key", "=", "value", "other", "=", "value"]
    );
}

#[test]
fn generic_css_ids_are_not_treated_as_hash_comments() {
    let options = Options {
        mode: crate::cli::Mode::Weak,
        ..Options::default()
    };
    let tokens = tokenize_for_detection("#app .title\n", "css", &options);

    assert_eq!(tokens.len(), 2);
}

#[test]
fn css_like_tokenizer_splits_punctuation() {
    let content = "#app .title { color: saturate(@base, 5%); }";
    let tokens = tokenize_for_detection(content, "css", &Options::default());
    let token_values = token_slices(content, &tokens);

    assert_eq!(
        token_values,
        vec![
            "#app", ".title", "{", "color", ":", "saturate", "(", "@base", ",", "5%", ")", ";", "}"
        ]
    );
}

#[test]
fn code_like_tokenizer_splits_punctuation_and_operators() {
    let content = "fn call<T>(x: i32) -> bool { x >= 1 }";
    let tokens = tokenize_for_detection(content, "rust", &Options::default());
    let token_values = token_slices(content, &tokens);

    assert_eq!(
        token_values,
        vec![
            "fn", "call", "<", "T", ">", "(", "x", ":", "i32", ")", "->", "bool", "{", "x", ">=",
            "1", "}"
        ]
    );
}

#[test]
fn long_tail_code_like_formats_split_punctuation_and_operators() {
    let content = "value = call(item, 1);";
    for format in [
        "aspnet",
        "cfml",
        "cfscript",
        "clojure",
        "cmake",
        "coffeescript",
        "csv",
        "dot",
        "eiffel",
        "haml",
        "ini",
        "markup",
        "ocaml",
        "plsql",
        "purescript",
        "python",
        "qsharp",
        "rescript",
        "robotframework",
        "sparql",
        "tt2",
        "yaml",
    ] {
        let tokens = tokenize_for_detection(content, format, &Options::default());
        let token_values = token_slices(content, &tokens);

        assert_eq!(
            token_values,
            vec!["value", "=", "call", "(", "item", ",", "1", ")", ";"],
            "{format}"
        );
    }
}

#[test]
fn twig_mild_mode_keeps_prism_default_whitespace() {
    let content = "<p>a</p>\n <p>b</p>\n{% include \"helper.html\" %}\n";
    let tokens = tokenize_for_detection(content, "twig", &Options::default());
    let values = token_slices(content, &tokens);

    assert!(values.contains(&"\n "));
    assert!(values.contains(&" "));
    assert!(!values.contains(&"\n"));
}

#[test]
fn weak_mode_skips_js_comments() {
    let options = Options {
        mode: crate::cli::Mode::Weak,
        ..Options::default()
    };
    let strong = tokenize_for_detection(
        "const a = 1; // comment\n",
        "javascript",
        &Options::default(),
    );
    let weak = tokenize_for_detection("const a = 1; // comment\n", "javascript", &options);
    assert!(strong.len() > weak.len());
}

#[test]
fn js_line_comments_split_into_comment_tokens() {
    let content = "// really an argument\nconst a = 1;\n";
    let tokens = tokenize_for_detection(content, "javascript", &Options::default());
    let comment_values = tokens
        .iter()
        .filter_map(|token| {
            let value = &content[token.range[0]..token.range[1]];
            value.starts_with("//").then_some(value)
        })
        .collect::<Vec<_>>();

    assert_eq!(comment_values, vec!["//"]);
    assert!(
        tokens
            .iter()
            .any(|token| &content[token.range[0]..token.range[1]] == "really")
    );
}

#[test]
fn js_hashbang_splits_like_prism() {
    let content = "#!/usr/bin/env node\n'use strict';\n";
    let tokens = tokenize_for_detection(content, "javascript", &Options::default());
    let values = token_slices(content, &tokens)
        .into_iter()
        .take(9)
        .collect::<Vec<_>>();

    assert_eq!(
        values,
        vec!["#", "!", "/", "usr", "/", "bin", "/", "env", "node"]
    );
    assert_eq!(tokens[0].hash, hash_token(TokenKind::Default, "#", false));
    assert_eq!(tokens[1].hash, hash_token(TokenKind::Operator, "!", false));
}

#[test]
fn splits_template_interpolation_like_prism() {
    let tokens = tokenize_for_detection(
        "const x = `a${b}c${d}e`;",
        "typescript",
        &Options::default(),
    );
    assert_eq!(tokens.len(), 13);
    assert_eq!(tokens[3].start.column, 11);
    assert_eq!(tokens[4].start.column, 13);
    assert_eq!(tokens[6].start.column, 16);
    assert_eq!(tokens[8].start.column, 18);
    assert_eq!(tokens[10].start.column, 21);
    assert_eq!(tokens[11].start.column, 22);
}

#[test]
fn keeps_template_interpolation_space_tokens_like_prism() {
    let content = "const x = `${store ? '[Store]' : '[No Store]'}`;";
    let tokens = tokenize_for_detection(content, "typescript", &Options::default());
    let values = token_slices(content, &tokens);

    assert!(
        values
            .windows(3)
            .any(|window| window == ["store", " ", "?"])
    );
    assert!(
        values
            .windows(3)
            .any(|window| window == ["?", " ", "'[Store]'"])
    );
    assert!(
        values
            .windows(3)
            .any(|window| window == ["'[Store]'", " ", ":"])
    );
    assert!(
        values
            .windows(3)
            .any(|window| window == [":", " ", "'[No Store]'"])
    );
}

#[test]
fn splits_optional_chaining_like_prism() {
    let tokens = tokenize_for_detection("a?.b", "typescript", &Options::default());
    assert_eq!(tokens.len(), 4);
    assert_eq!(tokens[1].start.column, 2);
    assert_eq!(tokens[2].start.column, 3);
    assert_eq!(tokens[3].start.column, 4);
}

#[test]
fn merges_adjacent_generic_closing_angles_like_prism() {
    let tokens = tokenize_for_detection("type A = X<Y<Z>>;", "typescript", &Options::default());
    assert_eq!(tokens.len(), 10);
    assert_eq!(tokens[8].start.column, 15);
    assert_eq!(tokens[8].end.column, 17);
    assert_eq!(tokens[9].start.column, 17);
}

#[test]
fn js_regex_after_recoverable_parse_error_stays_single_token() {
    let content = "export type Flags = {enabled: boolean};\n\
export function normalize(str) {\n\
  return str.replace(/Check your code at .+?:\\d+/g, 'Check your code at **');\n\
}\n";
    let tokens = tokenize_for_detection(content, "javascript", &Options::default());
    let values = token_slices(content, &tokens);

    assert!(values.contains(&"/Check your code at .+?:\\d+/g"));
    assert!(!values.windows(2).any(|window| window == ["/", "Check"]));
}

#[test]
fn typescript_array_regex_splits_like_prism() {
    let content = r#"const restrictions = [/\.css$/i];"#;
    let tokens = tokenize_for_detection(content, "typescript", &Options::default());
    let values = token_slices(content, &tokens);

    assert!(
        values
            .windows(6)
            .any(|window| window == ["/", "\\", ".", "css$", "/", "i"])
    );
    assert!(!values.contains(&r#"/\.css$/i"#));
}

#[test]
fn js_division_after_identifier_is_not_recovered_as_regex() {
    let content = "const ratio = total / count / scale;\n";
    let tokens = tokenize_for_detection(content, "javascript", &Options::default());
    let values = token_slices(content, &tokens);

    assert!(values.contains(&"/"));
    assert!(!values.contains(&"/ count /"));
}

#[test]
fn weak_mode_skips_generic_markup_comments() {
    let content = "<!-- comment -->\nalpha beta\n<!-- another -->\ngamma\n";
    let weak_options = Options {
        mode: crate::cli::Mode::Weak,
        ..Options::default()
    };

    let strong = tokenize_for_detection(content, "markup", &Options::default());
    let weak = tokenize_for_detection(content, "markup", &weak_options);

    assert_eq!(strong.len(), 5);
    assert_eq!(weak.len(), 3);
    let token_values = token_slices(content, &weak);
    assert_eq!(token_values, vec!["alpha", "beta", "gamma"]);
}