lindera-python 3.0.7

A Python binding for Lindera.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

from lindera import load_dictionary
from lindera import Tokenizer


def main():
    # load the dictionary
    dictionary = load_dictionary("embedded://ipadic")

    # create a tokenizer
    tokenizer = Tokenizer(dictionary, mode="decompose")

    text = "関西国際空港限定トートバッグを東京スカイツリーの最寄り駅であるとうきょうスカイツリー駅で買う"
    print(f"text: {text}\n")

    # tokenize the text
    tokens = tokenizer.tokenize(text)

    for token in tokens:
        print(token.surface)


if __name__ == "__main__":
    main()