seams 0.1.17

High-throughput sentence extractor for Project Gutenberg texts with dialog-aware detection
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198

// use std::path::PathBuf;
use seams::{discovery, sentence_detector};

#[path = "integration/mod.rs"]
mod test_utils;
use test_utils::{TestFixture};

/// Test pipeline behavior with malformed UTF-8 files
#[tokio::test]
async fn test_pipeline_invalid_utf8() {
    let fixture = TestFixture::new();
    
    // Create file with invalid UTF-8 bytes
    let invalid_path = fixture.root_path.join("invalid-0.txt");
    std::fs::write(&invalid_path, [0xFF, 0xFE, 0xFD]).expect("Failed to write invalid UTF-8 file");
    
    // Discovery should find the file (validation happens during processing now)
    let files = discovery::find_gutenberg_files(&fixture.root_path).await
        .expect("Discovery should succeed");
    
    // Should find the file during discovery (UTF-8 validation is now deferred)
    assert_eq!(files.len(), 1, "Files should be discovered regardless of UTF-8 content");
}

/// Test pipeline with permission denied scenarios
#[tokio::test]
async fn test_pipeline_permission_denied() {
    let fixture = TestFixture::new();
    let file_path = fixture.create_gutenberg_file("restricted-0.txt", "Test content.");
    
    // Remove read permissions (Unix-specific test)
    #[cfg(unix)]
    {
        use std::os::unix::fs::PermissionsExt;
        let mut perms = std::fs::metadata(&file_path).unwrap().permissions();
        perms.set_mode(0o000);
        std::fs::set_permissions(&file_path, perms).unwrap();
        
        // Discovery should find the file (access errors happen during processing now)
        let files = discovery::find_gutenberg_files(&fixture.root_path).await
            .expect("Discovery should handle permission errors");
        
        // Should find the file during discovery (access validation is now deferred)
        assert_eq!(files.len(), 1, "Files should be discovered regardless of access permissions");
        
        // Restore permissions for cleanup
        let mut perms = std::fs::metadata(&file_path).unwrap().permissions();
        perms.set_mode(0o644);
        std::fs::set_permissions(&file_path, perms).unwrap();
    }
}

/// Test pipeline with empty files
#[tokio::test]
async fn test_pipeline_empty_files() {
    let fixture = TestFixture::new();
    let file_path = fixture.create_gutenberg_file("empty-0.txt", "");
    
    // Discovery should find the file
    let files = discovery::find_gutenberg_files(&fixture.root_path).await
        .expect("Discovery should succeed");
    assert_eq!(files.len(), 1);
    
    // Reader should handle empty file
    let content = tokio::fs::read_to_string(&file_path).await
        .expect("Reading empty file should succeed");
    assert_eq!(content, "");
    
    // Sentence detector should handle empty content
    let detector = sentence_detector::dialog_detector::SentenceDetectorDialog::new()
        .expect("Detector creation should succeed");
    let sentences = detector.detect_sentences_borrowed(&content)
        .expect("Sentence detection on empty content should succeed");
    
    assert_eq!(sentences.len(), 0, "Empty file should produce no sentences");
}

/// Test pipeline with files containing only whitespace
#[tokio::test]
async fn test_pipeline_whitespace_only() {
    let fixture = TestFixture::new();
    let whitespace_content = "   \n\t  \r\n   ";
    let file_path = fixture.create_gutenberg_file("whitespace-0.txt", whitespace_content);
    
    let files = discovery::find_gutenberg_files(&fixture.root_path).await
        .expect("Discovery should succeed");
    assert_eq!(files.len(), 1);
    
    let content = tokio::fs::read_to_string(&file_path).await
        .expect("Reading whitespace file should succeed");
    
    let detector = sentence_detector::dialog_detector::SentenceDetectorDialog::new()
        .expect("Detector creation should succeed");
    let sentences = detector.detect_sentences_borrowed(&content)
        .expect("Sentence detection on whitespace should succeed");
    
    assert_eq!(sentences.len(), 0, "Whitespace-only file should produce no sentences");
}

/// Test pipeline with files containing only punctuation
#[tokio::test]
async fn test_pipeline_punctuation_only() {
    let fixture = TestFixture::new();
    let punct_content = "...!?!...";
    let file_path = fixture.create_gutenberg_file("punct-0.txt", punct_content);
    
    let files = discovery::find_gutenberg_files(&fixture.root_path).await
        .expect("Discovery should succeed");
    assert_eq!(files.len(), 1);
    
    let content = tokio::fs::read_to_string(&file_path).await
        .expect("Reading punctuation file should succeed");
    
    let detector = sentence_detector::dialog_detector::SentenceDetectorDialog::new()
        .expect("Detector creation should succeed");
    let sentences = detector.detect_sentences_borrowed(&content)
        .expect("Sentence detection on punctuation should succeed");
    
    // Should handle gracefully - might produce sentences depending on FST rules
    assert!(sentences.len() <= 3, "Punctuation-only should produce few or no sentences");
}

/// Test pipeline with non-matching filename patterns
#[tokio::test]
async fn test_pipeline_non_matching_files() {
    let fixture = TestFixture::new();
    
    // Create files that shouldn't match the pattern
    fixture.create_gutenberg_file("book-1.txt", "This should not match.");
    fixture.create_gutenberg_file("book.txt", "This should not match either.");
    fixture.create_gutenberg_file("readme.md", "Not a Gutenberg file.");
    
    // Create one matching file
    fixture.create_gutenberg_file("valid-0.txt", "This should match.");
    
    let files = discovery::find_gutenberg_files(&fixture.root_path).await
        .expect("Discovery should succeed");
    
    // Should only find the matching file
    assert_eq!(files.len(), 1);
    assert!(files[0].file_name().unwrap().to_str().unwrap() == "valid-0.txt");
}

/// Test pipeline with nested directory structures
#[tokio::test]
async fn test_pipeline_nested_directories() {
    let fixture = TestFixture::new();
    
    // Create files in nested structure
    fixture.create_gutenberg_file("level1/book-0.txt", "First level.");
    fixture.create_gutenberg_file("level1/level2/book-0.txt", "Second level.");
    fixture.create_gutenberg_file("level1/level2/level3/deep-0.txt", "Deep nesting.");
    
    let files = discovery::find_gutenberg_files(&fixture.root_path).await
        .expect("Discovery should succeed");
    
    assert_eq!(files.len(), 3, "Should find files in all nested directories");
    
    // Verify each file can be processed
    let detector = sentence_detector::dialog_detector::SentenceDetectorDialog::new()
        .expect("Detector creation should succeed");
    
    for file_path in files {
        let content = tokio::fs::read_to_string(&file_path).await
            .expect("File reading should succeed");
        let sentences = detector.detect_sentences_borrowed(&content)
            .expect("Sentence detection should succeed");
        
        assert_eq!(sentences.len(), 1, "Each test file should have one sentence");
    }
}

/// Test pipeline with very long file paths
#[tokio::test]
async fn test_pipeline_long_paths() {
    let fixture = TestFixture::new();
    
    // Create a deeply nested structure with long names
    let long_path = "very_long_directory_name/another_very_long_directory_name/yet_another_long_name/final_directory";
    let file_path_str = format!("{long_path}/extremely_long_filename_that_tests_path_limits-0.txt");
    
    fixture.create_gutenberg_file(&file_path_str, "Content in deeply nested file.");
    
    let files = discovery::find_gutenberg_files(&fixture.root_path).await
        .expect("Discovery should handle long paths");
    assert_eq!(files.len(), 1);
    
    let content = tokio::fs::read_to_string(&files[0]).await
        .expect("Should read file with long path");
    
    let detector = sentence_detector::dialog_detector::SentenceDetectorDialog::new()
        .expect("Detector creation should succeed");
    let sentences = detector.detect_sentences_borrowed(&content)
        .expect("Sentence detection should succeed");
    
    assert_eq!(sentences.len(), 1);
    assert_eq!(sentences[0].normalize(), "Content in deeply nested file.");
}