icu_segmenter/
lib.rs

1// This file is part of ICU4X. For terms of use, please see the file
2// called LICENSE at the top level of the ICU4X source tree
3// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5// https://github.com/unicode-org/icu4x/blob/main/documents/process/boilerplate.md#library-annotations
6#![cfg_attr(not(any(test, doc)), no_std)]
7#![cfg_attr(
8    not(test),
9    deny(
10        clippy::indexing_slicing,
11        clippy::unwrap_used,
12        clippy::expect_used,
13        clippy::panic,
14    )
15)]
16#![warn(missing_docs)]
17
18//! Segment strings by lines, graphemes, words, and sentences.
19//!
20//! This module is published as its own crate ([`icu_segmenter`](https://docs.rs/icu_segmenter/latest/icu_segmenter/))
21//! and as part of the [`icu`](https://docs.rs/icu/latest/icu/) crate. See the latter for more details on the ICU4X project.
22//!
23//! This module contains segmenter implementation for the following rules.
24//!
25//! - Line segmenter that is compatible with [Unicode Standard Annex #14][UAX14] (Version 15.1.0, or
26//!   Version 17.0.0 with the `*_17_*` and `*_neo_*` constructors) _Unicode Line Breaking Algorithm_, with options
27//!   to tailor line-breaking behavior for CSS [`line-break`] and [`word-break`] properties.
28//! - Grapheme cluster segmenter, word segmenter, and sentence segmenter that are compatible with
29//!   [Unicode Standard Annex #29][UAX29] (Version 17.0.0), _Unicode Text Segmentation_.
30//!
31//! [UAX14]: https://www.unicode.org/reports/tr14/tr14-51.html
32//! [UAX29]: https://www.unicode.org/reports/tr29/tr29-47.html
33//! [`line-break`]: https://drafts.csswg.org/css-text-3/#line-break-property
34//! [`word-break`]: https://drafts.csswg.org/css-text-3/#word-break-property
35//!
36//! # Examples
37//!
38//! ## Line Break
39//!
40//! Find line break opportunities:
41//!
42//!```rust
43//! use icu::segmenter::LineSegmenter;
44//!
45//! let segmenter = LineSegmenter::new_auto(Default::default());
46//!
47//! let breakpoints: Vec<usize> = segmenter
48//!     .segment_str("Hello World. Xin chào thế giới!")
49//!     .collect();
50//! assert_eq!(&breakpoints, &[0, 6, 13, 17, 23, 29, 36]);
51//! ```
52//!
53//! See [`LineSegmenter`] for more examples.
54//!
55//! ## Grapheme Cluster Break
56//!
57//! Find all grapheme cluster boundaries:
58//!
59//!```rust
60//! use icu::segmenter::GraphemeClusterSegmenter;
61//!
62//! let segmenter = GraphemeClusterSegmenter::new();
63//!
64//! let breakpoints: Vec<usize> = segmenter
65//!     .segment_str("Hello World. Xin chào thế giới!")
66//!     .collect();
67//! assert_eq!(
68//!     &breakpoints,
69//!     &[
70//!         0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
71//!         19, 21, 22, 23, 24, 25, 28, 29, 30, 31, 34, 35, 36
72//!     ]
73//! );
74//! ```
75//!
76//! See [`GraphemeClusterSegmenter`] for more examples.
77//!
78//! ## Word Break
79//!
80//! Find all word boundaries:
81//!
82//!```rust
83//! use icu::segmenter::{WordSegmenter, options::WordBreakInvariantOptions};
84//!
85//! let segmenter =
86//!     WordSegmenter::new_auto(WordBreakInvariantOptions::default());
87//!
88//! let breakpoints: Vec<usize> = segmenter
89//!     .segment_str("Hello World. Xin chào thế giới!")
90//!     .collect();
91//! assert_eq!(
92//!     &breakpoints,
93//!     &[0, 5, 6, 11, 12, 13, 16, 17, 22, 23, 28, 29, 35, 36]
94//! );
95//! ```
96//!
97//! See [`WordSegmenter`] for more examples.
98//!
99//! ## Sentence Break
100//!
101//! Segment the string into sentences:
102//!
103//!```rust
104//! use icu::segmenter::{
105//!     SentenceSegmenter, options::SentenceBreakInvariantOptions,
106//! };
107//!
108//! let segmenter =
109//!     SentenceSegmenter::new(SentenceBreakInvariantOptions::default());
110//!
111//! let breakpoints: Vec<usize> = segmenter
112//!     .segment_str("Hello World. Xin chào thế giới!")
113//!     .collect();
114//! assert_eq!(&breakpoints, &[0, 13, 36]);
115//! ```
116//!
117//! See [`SentenceSegmenter`] for more examples.
118
119extern crate alloc;
120
121mod complex;
122mod indices;
123mod rule_segmenter_v1;
124#[cfg(feature = "unstable")]
125mod rule_segmenter_v2;
126
127/// [`GraphemeClusterSegmenter`] and its related iterators, borrowed types, and options.
128mod grapheme;
129/// [`LineSegmenter`] and its related iterators, borrowed types, and options.
130mod line;
131/// [`SentenceSegmenter`] and its related iterators, borrowed types, and options.
132mod sentence;
133/// [`WordSegmenter`] and its related iterators, borrowed types, and options.
134mod word;
135
136pub mod provider;
137
138// Main Segmenter and BreakIterator public types
139pub use crate::grapheme::GraphemeClusterSegmenter;
140pub use crate::grapheme::GraphemeClusterSegmenterBorrowed;
141pub use crate::line::LineSegmenter;
142pub use crate::line::LineSegmenterBorrowed;
143pub use crate::sentence::SentenceSegmenter;
144pub use crate::sentence::SentenceSegmenterBorrowed;
145pub use crate::word::WordSegmenter;
146pub use crate::word::WordSegmenterBorrowed;
147
148/// Options structs and enums
149pub mod options {
150    pub use crate::line::LineBreakOptions;
151    pub use crate::line::LineBreakStrictness;
152    pub use crate::line::LineBreakWordOption;
153    pub use crate::sentence::SentenceBreakInvariantOptions;
154    pub use crate::sentence::SentenceBreakOptions;
155    pub use crate::word::WordBreakInvariantOptions;
156    pub use crate::word::WordBreakOptions;
157    pub use crate::word::WordType;
158}
159
160/// Largely-internal scaffolding types (You should very rarely need to reference these directly)
161// TODO: These should have never been public
162pub mod scaffold;
163
164/// Types supporting iteration over segments. Obtained from the segmenter types.
165pub mod iterators {
166    pub use crate::grapheme::GraphemeClusterBreakIterator;
167    pub use crate::line::LineBreakIterator;
168    pub use crate::sentence::SentenceBreakIterator;
169    pub use crate::word::{WordBreakIterator, WordBreakIteratorWithWordType};
170}
171
172pub(crate) mod private {
173    /// Trait marking other traits that are considered unstable and should not generally be
174    /// implemented outside of the segmenter crate.
175    pub trait Sealed {}
176}
icu_segmenter/lib.rs

icu_segmenter/
lib.rs
