Struct TokenizeOptions

Source

pub struct TokenizeOptions {
    pub mode: Mode,
    pub ignore_case: bool,
    pub ignore_ranges: Vec<[usize; 2]>,
    pub code_ignore_regexes: Vec<Regex>,
}

Expand description

Options for the detection-path tokenizer.

Carries mode, case-folding flag, pre-parsed ignore-region byte ranges, and pre-compiled code-level regex patterns that skip matching tokens during detection.

Code-level ignore patterns (v4 ignorePattern) work by matching regex patterns against source text, collecting byte ranges of matches, and then filtering any token whose byte range overlaps a match — identical in effect to v4’s setupIgnorePatterns which injected Prism grammar tokens.

Fields§

§mode: Mode§ignore_case: bool

When true, token values are lowercased before hashing.

§ignore_ranges: Vec<[usize; 2]>

Ignored byte ranges from jscpd:ignore-start / jscpd:ignore-end and code-level regex matches from ignorePattern. Each entry is [start_byte, end_byte).

§code_ignore_regexes: Vec<Regex>

Pre-compiled code-level regex patterns inherited from v4 ignorePattern. Before tokenization, these are matched against the source text and overlapping byte ranges are added to ignore_ranges.

Struct TokenizeOptions Copy item path

Fields§

Implementations§

impl TokenizeOptions

pub fn new(mode: Mode) -> Self

pub fn with_code_ignore_patterns(mode: Mode, patterns: &[String]) -> Self

Trait Implementations§

impl Clone for TokenizeOptions

fn clone(&self) -> TokenizeOptions

fn clone_from(&mut self, source: &Self)

impl Debug for TokenizeOptions

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Auto Trait Implementations§

impl Freeze for TokenizeOptions

impl RefUnwindSafe for TokenizeOptions

impl Send for TokenizeOptions

impl Sync for TokenizeOptions

impl Unpin for TokenizeOptions

impl UnsafeUnpin for TokenizeOptions

impl UnwindSafe for TokenizeOptions

Blanket Implementations§

impl<T> Any for Twhere T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> CloneToUninit for Twhere T: Clone,

unsafe fn clone_to_uninit(&self, dest: *mut u8)

impl<T> From<T> for T

fn from(t: T) -> T

impl<'a, T> FromIn<'a, T> for T

fn from_in(t: T, _: &'a Allocator) -> T

impl<T, U> Into<U> for Twhere U: From<T>,

fn into(self) -> U

impl<T> IntoEither for T

fn into_either(self, into_left: bool) -> Either<Self, Self>

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>where F: FnOnce(&Self) -> bool,

impl<'a, T, U> IntoIn<'a, U> for Twhere U: FromIn<'a, T>,

fn into_in(self, allocator: &'a Allocator) -> U

impl<D> OwoColorize for D

fn fg<C>(&self) -> FgColorDisplay<'_, C, Self>where C: Color,

fn bg<C>(&self) -> BgColorDisplay<'_, C, Self>where C: Color,

fn black(&self) -> FgColorDisplay<'_, Black, Self>

fn on_black(&self) -> BgColorDisplay<'_, Black, Self>

fn red(&self) -> FgColorDisplay<'_, Red, Self>

fn on_red(&self) -> BgColorDisplay<'_, Red, Self>

fn green(&self) -> FgColorDisplay<'_, Green, Self>

fn on_green(&self) -> BgColorDisplay<'_, Green, Self>

fn yellow(&self) -> FgColorDisplay<'_, Yellow, Self>

fn on_yellow(&self) -> BgColorDisplay<'_, Yellow, Self>

fn blue(&self) -> FgColorDisplay<'_, Blue, Self>

fn on_blue(&self) -> BgColorDisplay<'_, Blue, Self>

fn magenta(&self) -> FgColorDisplay<'_, Magenta, Self>

fn on_magenta(&self) -> BgColorDisplay<'_, Magenta, Self>

fn purple(&self) -> FgColorDisplay<'_, Magenta, Self>

fn on_purple(&self) -> BgColorDisplay<'_, Magenta, Self>

fn cyan(&self) -> FgColorDisplay<'_, Cyan, Self>

fn on_cyan(&self) -> BgColorDisplay<'_, Cyan, Self>

fn white(&self) -> FgColorDisplay<'_, White, Self>

fn on_white(&self) -> BgColorDisplay<'_, White, Self>

fn default_color(&self) -> FgColorDisplay<'_, Default, Self>

fn on_default_color(&self) -> BgColorDisplay<'_, Default, Self>

fn bright_black(&self) -> FgColorDisplay<'_, BrightBlack, Self>

fn on_bright_black(&self) -> BgColorDisplay<'_, BrightBlack, Self>

fn bright_red(&self) -> FgColorDisplay<'_, BrightRed, Self>

fn on_bright_red(&self) -> BgColorDisplay<'_, BrightRed, Self>

fn bright_green(&self) -> FgColorDisplay<'_, BrightGreen, Self>

fn on_bright_green(&self) -> BgColorDisplay<'_, BrightGreen, Self>

fn bright_yellow(&self) -> FgColorDisplay<'_, BrightYellow, Self>

fn on_bright_yellow(&self) -> BgColorDisplay<'_, BrightYellow, Self>

fn bright_blue(&self) -> FgColorDisplay<'_, BrightBlue, Self>

fn on_bright_blue(&self) -> BgColorDisplay<'_, BrightBlue, Self>

fn bright_magenta(&self) -> FgColorDisplay<'_, BrightMagenta, Self>

fn on_bright_magenta(&self) -> BgColorDisplay<'_, BrightMagenta, Self>

fn bright_purple(&self) -> FgColorDisplay<'_, BrightMagenta, Self>

fn on_bright_purple(&self) -> BgColorDisplay<'_, BrightMagenta, Self>

fn bright_cyan(&self) -> FgColorDisplay<'_, BrightCyan, Self>

fn on_bright_cyan(&self) -> BgColorDisplay<'_, BrightCyan, Self>

fn bright_white(&self) -> FgColorDisplay<'_, BrightWhite, Self>

Struct TokenizeOptions

impl<T> Any for T
where T: 'static + ?Sized,

impl<T> Borrow<T> for T
where T: ?Sized,

impl<T> BorrowMut<T> for T
where T: ?Sized,

impl<T> CloneToUninit for T
where T: Clone,

impl<T, U> Into<U> for T
where U: From<T>,

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
where F: FnOnce(&Self) -> bool,

impl<'a, T, U> IntoIn<'a, U> for T
where U: FromIn<'a, T>,

fn fg<C>(&self) -> FgColorDisplay<'_, C, Self>
where C: Color,

fn bg<C>(&self) -> BgColorDisplay<'_, C, Self>
where C: Color,

fn color<Color>(&self, color: Color) -> FgDynColorDisplay<'_, Color, Self>
where Color: DynColor,

fn on_color<Color>(&self, color: Color) -> BgDynColorDisplay<'_, Color, Self>
where Color: DynColor,

impl<T> ToOwned for T
where T: Clone,

impl<T, U> TryFrom<U> for T
where U: Into<T>,

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,