sentencex 0.1.26

Sentence segmentation library with wide language support optimized for speed and utility.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71

use clap::Parser;
use sentencex::{get_sentence_boundaries, segment};
use std::fs;
use std::io::{self, Read};
use std::time::Instant;

/// CLI for Sentence Segmentation
#[derive(Parser, Debug)]
#[command(name = "sentencex")]
#[command(about = "A tool for sentence segmentation", long_about = None)]
#[clap(version)]
struct Cli {
    /// Path to input file
    #[arg(short, long)]
    file: Option<String>,

    /// The language of the text
    #[arg(short, long, default_value = "en")]
    language: String,

    /// Print debug information including boundary details
    #[arg(short, long)]
    debug: bool,
}

fn main() {
    let cli = Cli::parse();

    let text = match cli.file {
        Some(file_path) => fs::read_to_string(file_path).expect("Failed to read file"),
        None => {
            // Read from stdin when no file is provided
            let mut buffer = String::new();
            io::stdin()
                .read_to_string(&mut buffer)
                .expect("Failed to read from stdin");
            buffer
        }
    };

    if cli.debug {
        let start_time = Instant::now();
        let boundaries = get_sentence_boundaries(&cli.language, &text);
        let elapsed = start_time.elapsed();

        eprintln!("Time taken for get_sentence_boundaries(): {:?}", elapsed);

        for (i, boundary) in boundaries.iter().enumerate() {
            println!("Boundary {}: ", i + 1);
            println!("  Start Index: {}", boundary.start_index);
            println!("  End Index: {}", boundary.end_index);
            println!("  Start Byte: {}", boundary.start_byte);
            println!("  End Byte: {}", boundary.end_byte);

            println!("  Text: {:?}", boundary.text);
            println!("  Boundary Symbol: {:?}", boundary.boundary_symbol);
            println!("  Is Paragraph Break: {}", boundary.is_paragraph_break);
            println!();
        }
    } else {
        let start_time = Instant::now();
        let sentences = segment(&cli.language, &text);
        let elapsed = start_time.elapsed();
        for sentence in sentences.iter() {
            println!("* {}", sentence);
        }

        eprintln!("Time taken for segment(): {:?}", elapsed);
        eprintln!("Total sentences: {:?}", sentences.len());
    }
}