rustling 0.8.0

A blazingly fast library for computational linguistics
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

"""Tests for rustling.wordseg.LongestStringMatching."""

import pytest

from rustling.wordseg import LongestStringMatching


def test_basic():
    """Test basic segmentation from the original wordseg package."""
    model = LongestStringMatching(max_word_length=4)
    model.fit(
        [
            ("this", "is", "a", "sentence"),
            ("that", "is", "not", "a", "sentence"),
        ]
    )
    result = model.predict(["thatisadog", "thisisnotacat"])
    assert result == [
        ["that", "is", "a", "d", "o", "g"],
        ["this", "is", "not", "a", "c", "a", "t"],
    ]


def test_invalid_max_word_length():
    """Test that max_word_length < 2 raises ValueError."""
    with pytest.raises(ValueError, match="max_word_length must be >= 2"):
        LongestStringMatching(max_word_length=1)


def test_empty_input():
    """Test segmentation of empty string."""
    model = LongestStringMatching(max_word_length=4)
    model.fit([["hello", "world"]])
    result = model.predict([""])
    assert result == [[]]


def test_no_training_data():
    """Test segmentation with no training data falls back to single chars."""
    model = LongestStringMatching(max_word_length=4)
    model.fit([])
    result = model.predict(["hello"])
    assert result == [["h", "e", "l", "l", "o"]]


def test_single_char_words_ignored_in_training():
    """Test that single-character words are ignored in training."""
    model = LongestStringMatching(max_word_length=4)
    # Single-character words should be ignored in training
    model.fit([["a", "b", "ab"]])
    # "ab" should be recognized, but not single chars
    result = model.predict(["abab"])
    assert result == [["ab", "ab"]]


def test_unicode_chars():
    """Test segmentation of Unicode characters (e.g., Chinese)."""
    model = LongestStringMatching(max_word_length=4)
    model.fit([["你好", "世界"]])
    result = model.predict(["你好世界"])
    assert result == [["你好", "世界"]]


def test_max_word_length_constraint():
    """Test that max_word_length limits matching length."""
    model = LongestStringMatching(max_word_length=3)
    # Train with a word longer than max_word_length
    model.fit([["hello"]])
    # Even though "hello" is in training, we can only match up to 3 chars
    # "hel" is not in training, so we fall back character by character
    result = model.predict(["hello"])
    assert result == [["h", "e", "l", "l", "o"]]


def test_accepts_tuples_in_fit():
    """Test that fit accepts tuples (like the original Python wordseg)."""
    model = LongestStringMatching(max_word_length=4)
    model.fit(
        [
            ("this", "is"),
            ("that", "was"),
        ]
    )
    result = model.predict(["thisis"])
    assert result == [["this", "is"]]


def test_multiple_sentences():
    """Test segmentation of multiple sentences at once."""
    model = LongestStringMatching(max_word_length=4)
    model.fit([["the", "cat"]])
    result = model.predict(["thecat", "catthe", "the"])
    assert result == [["the", "cat"], ["cat", "the"], ["the"]]


def test_predict_offsets():
    """Test predict with offsets=True."""
    model = LongestStringMatching(max_word_length=4)
    model.fit([["你好", "世界"]])
    input_str = "你好世界"
    result = model.predict([input_str], offsets=True)
    assert result == [[("你好", (0, 2)), ("世界", (2, 4))]]


def test_predict_offsets_ascii():
    """Test predict with offsets=True for ASCII input."""
    model = LongestStringMatching(max_word_length=4)
    model.fit([["this", "is"]])
    input_str = "thisis"
    result = model.predict([input_str], offsets=True)
    assert len(result) == 1
    for word, (start, end) in result[0]:
        assert input_str[start:end] == word


def test_predict_offsets_false_unchanged():
    """Test that offsets=False returns plain strings."""
    model = LongestStringMatching(max_word_length=4)
    model.fit([["hello", "world"]])
    result = model.predict(["helloworld"], offsets=False)
    assert all(isinstance(w, str) for w in result[0])


def test_predict_offsets_empty():
    """Test predict with offsets=True on empty input."""
    model = LongestStringMatching(max_word_length=4)
    model.fit([])
    assert model.predict([], offsets=True) == []
    assert model.predict([""], offsets=True) == [[]]