Struct Trainer 

pub struct Trainer { /* private fields */ }

CRF-based morphological analysis trainer for Japanese text

This trainer implements standard CRF-based morphological analysis training, adapted for Lindera’s architecture. It supports:

• Feature extraction from vocabulary and corpus
• L-BFGS optimization for weight learning
• Unknown word categorization for 6 character types
• Connection cost matrix generation

Training Process

1. Initialize feature provider with vocabulary entries
2. Build lattices from training corpus
3. Execute CRF training with L-BFGS optimization
4. Extract learned weights and create final model

Example

use lindera_trainer::{Trainer, TrainerConfig};
use std::io::Cursor;

// Create minimal training data for demonstration
let seed_csv = "これ,0,0,1000,連体詞,*,*,*,*,*,これ,コレ,コレ\n";
let char_def = "DEFAULT 0 1 0\nHIRAGANA 1 1 0\n0x3042..0x3096 HIRAGANA\n";
let unk_def = "DEFAULT,0,0,1500,名詞,一般,*,*,*,*,*,*,*\n";
let feature_def = "UNIGRAM:%F[0]\nUNIGRAM:%F[1]\n";
let rewrite_def = "*\tUNK\n";

let config = TrainerConfig::from_readers(
    Cursor::new(seed_csv),
    Cursor::new(char_def),
    Cursor::new(unk_def),
    Cursor::new(feature_def),
    Cursor::new(rewrite_def)
).unwrap();

let trainer = Trainer::new(config).unwrap()
    .regularization_cost(0.01)
    .max_iter(10); // Reduced for doc test

// Note: In practice, you would load an actual corpus file
// let corpus = Corpus::from_reader(corpus_reader).unwrap();
// let model = trainer.train(corpus).unwrap();

Implementations

impl Trainer

pub fn new(config: TrainerConfig) -> Result<Self>

Creates a new Trainer using the specified configuration.

pub fn regularization_cost(self, cost: f64) -> Self

Sets the regularization cost coefficient.

pub fn use_l2(self, l2: bool) -> Self

Sets whether to use L2 regularization (default: L1).

pub fn elastic_net_l1_ratio(self, ratio: f64) -> Self

Sets the Elastic Net L1 ratio (0.0 to 1.0). When set, uses Elastic Net regularization instead of pure L1 or L2. l1_ratio=1.0 is pure L1, l1_ratio=0.0 is pure L2.

pub fn max_iter(self, iter: u64) -> Self

Sets the maximum number of iterations.

pub fn num_threads(self, threads: usize) -> Self

Sets the number of threads for training.

pub fn max_grouping_len(self, len: Option<usize>) -> Self

Sets the maximum grouping length for token sequences.

pub fn get_regularization_cost(&self) -> f64

Get the regularization cost (lambda)

pub fn get_max_iter(&self) -> u64

Get the maximum number of iterations

pub fn get_num_threads(&self) -> usize

Get the number of threads

pub fn train(self, corpus: Corpus) -> Result<Model>

Trains a model from the given corpus.