lindera-ruby 3.0.0

A Ruby binding for Lindera.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

# frozen_string_literal: true

require_relative 'test_helper'

class TestToken < Minitest::Test
  def setup
    dictionary = Lindera.load_dictionary('embedded://ipadic')
    tokenizer = Lindera::Tokenizer.new(dictionary, 'normal', nil)
    @tokens = tokenizer.tokenize('関西国際空港')
  end

  def test_token_attributes
    token = @tokens[0]

    assert_kind_of Lindera::Token, token
    assert_kind_of String, token.surface
    assert_kind_of Integer, token.byte_start
    assert_kind_of Integer, token.byte_end
    assert_kind_of Integer, token.position
    assert_kind_of Integer, token.word_id
  end

  def test_token_details
    token = @tokens[0]
    details = token.details

    assert_kind_of Array, details
    assert details.length.positive?

    first_detail = token.get_detail(0)
    assert_equal details[0], first_detail
  end

  def test_get_detail_out_of_bounds
    token = @tokens[0]
    assert_nil token.get_detail(9999)
  end

  def test_token_to_s
    token = @tokens[0]
    assert_equal token.surface, token.to_s
  end

  def test_token_inspect
    token = @tokens[0]
    inspect_str = token.inspect
    assert inspect_str.include?('Lindera::Token')
    assert inspect_str.include?(token.surface)
  end
end