Struct punkt::params::Standard

pub struct Standard;

Standard settings for all tokenizers, and trainers.

Trait Implementations

impl DefinesSentenceEndings for Standard

const SENTENCE_ENDINGS: &'static Set<char>

The set of characters that constitute a sentence ending.

fn is_sentence_ending(c: &char) -> bool

Checks if a character is a sentence ending.

impl DefinesInternalPunctuation for Standard

const INTERNAL_PUNCTUATION: &'static Set<char>

The set of legal punctuation characters that can occur within a word.

fn is_internal_punctuation(c: &char) -> bool

Checks if a character is a legal punctuation character that can occur within a word.

impl DefinesNonWordCharacters for Standard

const NONWORD_CHARS: &'static Set<char>

The set of characters that can not occur inside of a word.

fn is_nonword_char(c: &char) -> bool

Checks if a character is one that can not occur inside of a word.

impl DefinesPunctuation for Standard

const PUNCTUATION: &'static Set<char>

The set of legal punctuation marks.

fn is_punctuation(c: &char) -> bool

Checks if a characters is a legal punctuation mark.

impl DefinesNonPrefixCharacters for Standard

const NONPREFIX_CHARS: &'static Set<char>

The set of characters that can not start a word.

fn is_nonprefix_char(c: &char) -> bool

Checks if a character can start a word.

impl TrainerParameters for Standard

const ABBREV_LOWER_BOUND: f64

Lower bound score for a token to be considered an abbreviation.

const ABBREV_UPPER_BOUND: f64

Upper bound score for a token to be considered an abbreviation.

const IGNORE_ABBREV_PENALTY: bool

Disables the abbreviation penalty which exponentially penalizes occurances of words without a trailing period.

const COLLOCATION_LOWER_BOUND: f64

Lower bound score for two tokens to be considered a collocation

const SENTENCE_STARTER_LOWER_BOUND: f64

Lower bound score for a token to be considered a sentence starter.

const INCLUDE_ALL_COLLOCATIONS: bool

Include all pairs where the first token ends with a period.

const INCLUDE_ABBREV_COLLOCATIONS: bool

Include all pairs where the first is an abbreviation. Overridden by include_all_collocations.

const COLLOCATION_FREQUENCY_LOWER_BOUND: f64

Minimum number of times a bigram appears in order to be considered a collocation.