lindera-ruby 3.0.4

A Ruby binding for Lindera.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

# frozen_string_literal: true

require_relative 'test_helper'

class TestTokenizeIpadic < Minitest::Test
  def setup
    @dictionary = Lindera.load_dictionary('embedded://ipadic')
  end

  def test_basic_tokenize
    tokenizer = Lindera::Tokenizer.new(@dictionary, 'normal', nil)
    tokens = tokenizer.tokenize('すもももももももものうち')

    assert_equal 7, tokens.length
    assert_equal 'すもも', tokens[0].surface
    assert_equal 'も', tokens[1].surface
    assert_equal 'もも', tokens[2].surface
    assert_equal 'も', tokens[3].surface
    assert_equal 'もも', tokens[4].surface
    assert_equal 'の', tokens[5].surface
    assert_equal 'うち', tokens[6].surface
  end

  def test_tokenize_with_builder
    builder = Lindera::TokenizerBuilder.new
    builder.set_mode('normal')
    builder.set_dictionary('embedded://ipadic')
    tokenizer = builder.build

    tokens = tokenizer.tokenize('関西国際空港')
    assert tokens.length.positive?
    assert_equal '関西国際空港', tokens.map(&:surface).join
  end

  def test_tokenize_nbest
    tokenizer = Lindera::Tokenizer.new(@dictionary, 'normal', nil)
    results = tokenizer.tokenize_nbest('すもももももももものうち', 3, false, nil)

    assert results.length.positive?
    results.each do |pair|
      tokens = pair[0]
      cost = pair[1]
      assert_kind_of Array, tokens
      assert_kind_of Integer, cost
    end
  end
end