use alloc::vec::Vec;
use core::ops::Range;
use icu_provider::DataProvider;
use icu_segmenter::provider::{SegmenterDictionaryAutoV1, SegmenterDictionaryExtendedV1};
use icu_locale_core::{LanguageIdentifier, langid};
use icu_properties::props::{BidiMirroringGlyph, GeneralCategory, GraphemeClusterBreak, Script};
use icu_properties::{CodePointMapData, CodePointMapDataBorrowed};
use icu_segmenter::options::{
LineBreakOptions, LineBreakStrictness, LineBreakWordOption, WordBreakInvariantOptions,
};
use icu_segmenter::{
GraphemeClusterSegmenter, GraphemeClusterSegmenterBorrowed, LineSegmenter,
LineSegmenterBorrowed, WordSegmenter,
};
use parlance::{BaseDirection, BidiLevel, LineBreak, WordBreak};
use parley_data::Properties;
use crate::bidi;
use crate::break_overrides::LineBreakContext;
use crate::{AnalysisOptions, Analyzer, LineBreakConfig};
#[derive(Debug, Default)]
pub struct Analysis {
pub(crate) info: Vec<CharInfo>,
pub(crate) levels: Vec<BidiLevel>,
pub(crate) paragraph_level: BidiLevel,
}
impl Analysis {
#[inline]
pub fn new() -> Self {
Self::default()
}
pub(crate) fn clear(&mut self) {
self.info.clear();
self.levels.clear();
self.paragraph_level = BidiLevel::new(0);
}
#[inline(always)]
pub fn char_info(&self) -> &[CharInfo] {
&self.info
}
#[inline(always)]
pub fn bidi_levels(&self) -> &[BidiLevel] {
&self.levels
}
#[inline(always)]
pub fn paragraph_level(&self) -> BidiLevel {
self.paragraph_level
}
}
pub trait DictionaryProvider:
DataProvider<SegmenterDictionaryAutoV1> + DataProvider<SegmenterDictionaryExtendedV1>
{
}
impl<P: ?Sized> DictionaryProvider for P where
P: DataProvider<SegmenterDictionaryAutoV1> + DataProvider<SegmenterDictionaryExtendedV1>
{
}
#[derive(Default)]
pub struct AnalysisDataSources<'a> {
provider: Option<&'a dyn DictionaryProvider>,
}
impl core::fmt::Debug for AnalysisDataSources<'_> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("AnalysisDataSources")
.field("external_dictionaries", &self.provider.is_some())
.finish()
}
}
impl<'a> AnalysisDataSources<'a> {
pub const fn new(provider: Option<&'a dyn DictionaryProvider>) -> Self {
Self { provider }
}
#[inline(always)]
fn has_word_dictionary(&self, script: Script) -> bool {
(self.provider.is_some() || cfg!(feature = "complex-scripts"))
&& matches!(
script,
Script::Han
| Script::Hiragana
| Script::Thai
| Script::Lao
| Script::Khmer
| Script::Myanmar
)
}
#[inline(always)]
pub(crate) fn properties(&self, c: char) -> Properties {
Properties::get(c)
}
#[inline(always)]
fn grapheme_segmenter(&self) -> GraphemeClusterSegmenterBorrowed<'_> {
const { GraphemeClusterSegmenter::new() }
}
#[inline]
fn word_segmenter(&self) -> WordSegmenter {
if let Some(provider) = self.provider {
let mut segmenter =
WordSegmenter::new_for_non_complex_scripts(WordBreakInvariantOptions::default())
.static_to_owned();
segmenter.load_dictionary_unstable(provider).unwrap();
return segmenter;
}
#[cfg(feature = "complex-scripts")]
{
WordSegmenter::new_dictionary(WordBreakInvariantOptions::default()).static_to_owned()
}
#[cfg(not(feature = "complex-scripts"))]
{
const { WordSegmenter::new_for_non_complex_scripts(WordBreakInvariantOptions::default()) }
.static_to_owned()
}
}
#[inline]
fn line_segmenter(&self, key: SegmenterKey) -> LineSegmenter {
let options = Self::line_break_options(key);
match self.provider {
Some(provider) => {
let mut segmenter =
LineSegmenter::new_for_non_complex_scripts(options).static_to_owned();
segmenter.load_dictionary_unstable(provider).unwrap();
segmenter
}
None => line_segmenter_impl(options).static_to_owned(),
}
}
fn line_break_options(key: SegmenterKey) -> LineBreakOptions<'static> {
static JA: LanguageIdentifier = langid!("ja");
let mut opt = LineBreakOptions::default();
opt.word_option = Some(key.word_option);
opt.strictness = Some(key.strictness);
opt.content_locale = key.ja_zh.then_some(&JA);
opt
}
#[inline(always)]
fn brackets(&self) -> CodePointMapDataBorrowed<'_, BidiMirroringGlyph> {
const { CodePointMapData::new() }
}
}
#[cfg(feature = "complex-scripts")]
#[inline(always)]
fn line_segmenter_impl(opt: LineBreakOptions<'_>) -> LineSegmenterBorrowed<'static> {
LineSegmenter::new_dictionary(opt)
}
#[cfg(not(feature = "complex-scripts"))]
#[inline(always)]
fn line_segmenter_impl(opt: LineBreakOptions<'_>) -> LineSegmenterBorrowed<'static> {
LineSegmenter::new_for_non_complex_scripts(opt)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct SegmenterKey {
word_option: LineBreakWordOption,
strictness: LineBreakStrictness,
ja_zh: bool,
}
impl SegmenterKey {
const DEFAULT: Self = Self {
word_option: LineBreakWordOption::Normal,
strictness: LineBreakStrictness::Strict,
ja_zh: false,
};
fn new(config: &LineBreakConfig) -> Self {
let strictness = match config.line_break {
LineBreak::Strict => LineBreakStrictness::Strict,
LineBreak::Normal => LineBreakStrictness::Normal,
LineBreak::Loose => LineBreakStrictness::Loose,
LineBreak::Anywhere => return Self::DEFAULT,
};
let word_option = match config.word_break {
WordBreak::Normal => LineBreakWordOption::Normal,
WordBreak::BreakAll => LineBreakWordOption::BreakAll,
WordBreak::KeepAll => LineBreakWordOption::KeepAll,
};
let ja_zh = match strictness {
LineBreakStrictness::Normal | LineBreakStrictness::Loose => config
.language
.is_some_and(|language| matches!(language.language(), "ja" | "zh")),
_ => false,
};
Self {
word_option,
strictness,
ja_zh,
}
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub struct CharInfo {
pub script: Script,
flags: u16,
}
impl CharInfo {
const VARIATION_SELECTOR_SHIFT: u16 = 0;
const REGION_INDICATOR_SHIFT: u16 = 1;
const CONTROL_SHIFT: u16 = 2;
const EMOJI_OR_PICTOGRAPH_SHIFT: u16 = 3;
const CONTRIBUTES_TO_SHAPING_SHIFT: u16 = 4;
const FORCE_NORMALIZE_SHIFT: u16 = 5;
const GRAPHEME_START_SHIFT: u16 = 6;
const WORD_BOUNDARY_SHIFT: u8 = 7;
const SOFT_WRAP_OPPORTUNITY_SHIFT: u16 = 8;
const EMOJI_SHIFT: u16 = 9;
const EMOJI_PRESENTATION_SHIFT: u16 = 10;
const EMOJI_MODIFIER_SHIFT: u16 = 11;
const EMOJI_MODIFIER_BASE_SHIFT: u16 = 12;
const INLINE_OBJECT_SHIFT: u16 = 13;
const DEFAULT_IGNORABLE_SHIFT: u16 = 14;
const VARIATION_SELECTOR_MASK: u16 = 1 << Self::VARIATION_SELECTOR_SHIFT;
const REGION_INDICATOR_MASK: u16 = 1 << Self::REGION_INDICATOR_SHIFT;
const CONTROL_MASK: u16 = 1 << Self::CONTROL_SHIFT;
const EMOJI_OR_PICTOGRAPH_MASK: u16 = 1 << Self::EMOJI_OR_PICTOGRAPH_SHIFT;
const CONTRIBUTES_TO_SHAPING_MASK: u16 = 1 << Self::CONTRIBUTES_TO_SHAPING_SHIFT;
const FORCE_NORMALIZE_MASK: u16 = 1 << Self::FORCE_NORMALIZE_SHIFT;
const GRAPHEME_START_MASK: u16 = 1 << Self::GRAPHEME_START_SHIFT;
const WORD_BOUNDARY_MASK: u16 = 1 << Self::WORD_BOUNDARY_SHIFT;
const SOFT_WRAP_OPPORTUNITY_MASK: u16 = 1 << Self::SOFT_WRAP_OPPORTUNITY_SHIFT;
const EMOJI_MASK: u16 = 1 << Self::EMOJI_SHIFT;
const EMOJI_PRESENTATION_MASK: u16 = 1 << Self::EMOJI_PRESENTATION_SHIFT;
const EMOJI_MODIFIER_MASK: u16 = 1 << Self::EMOJI_MODIFIER_SHIFT;
const EMOJI_MODIFIER_BASE_MASK: u16 = 1 << Self::EMOJI_MODIFIER_BASE_SHIFT;
const INLINE_OBJECT_MASK: u16 = 1 << Self::INLINE_OBJECT_SHIFT;
const DEFAULT_IGNORABLE_MASK: u16 = 1 << Self::DEFAULT_IGNORABLE_SHIFT;
fn new(
script: Script,
is_variation_selector: bool,
is_region_indicator: bool,
is_control: bool,
is_emoji_or_pictograph: bool,
contributes_to_shaping: bool,
force_normalize: bool,
is_grapheme_start: bool,
is_word_boundary: bool,
is_soft_wrap_opportunity: bool,
is_emoji: bool,
is_emoji_presentation: bool,
is_emoji_modifier: bool,
is_emoji_modifier_base: bool,
is_default_ignorable: bool,
) -> Self {
Self {
script,
flags: (is_variation_selector as u16) << Self::VARIATION_SELECTOR_SHIFT
| (is_region_indicator as u16) << Self::REGION_INDICATOR_SHIFT
| (is_control as u16) << Self::CONTROL_SHIFT
| (is_emoji_or_pictograph as u16) << Self::EMOJI_OR_PICTOGRAPH_SHIFT
| (contributes_to_shaping as u16) << Self::CONTRIBUTES_TO_SHAPING_SHIFT
| (force_normalize as u16) << Self::FORCE_NORMALIZE_SHIFT
| (is_grapheme_start as u16) << Self::GRAPHEME_START_SHIFT
| (is_word_boundary as u16) << Self::WORD_BOUNDARY_SHIFT
| (is_soft_wrap_opportunity as u16) << Self::SOFT_WRAP_OPPORTUNITY_SHIFT
| (is_emoji as u16) << Self::EMOJI_SHIFT
| (is_emoji_presentation as u16) << Self::EMOJI_PRESENTATION_SHIFT
| (is_emoji_modifier as u16) << Self::EMOJI_MODIFIER_SHIFT
| (is_emoji_modifier_base as u16) << Self::EMOJI_MODIFIER_BASE_SHIFT
| (is_default_ignorable as u16) << Self::DEFAULT_IGNORABLE_SHIFT,
}
}
#[inline(always)]
pub fn is_variation_selector(self) -> bool {
self.flags & Self::VARIATION_SELECTOR_MASK != 0
}
#[inline(always)]
pub fn is_region_indicator(self) -> bool {
self.flags & Self::REGION_INDICATOR_MASK != 0
}
#[inline(always)]
pub fn is_control(self) -> bool {
self.flags & Self::CONTROL_MASK != 0
}
#[inline(always)]
pub fn is_emoji_or_pictograph(self) -> bool {
self.flags & Self::EMOJI_OR_PICTOGRAPH_MASK != 0
}
#[inline(always)]
pub fn is_emoji(self) -> bool {
self.flags & Self::EMOJI_MASK != 0
}
#[inline(always)]
pub fn is_emoji_presentation(self) -> bool {
self.flags & Self::EMOJI_PRESENTATION_MASK != 0
}
#[inline(always)]
pub fn is_emoji_modifier(self) -> bool {
self.flags & Self::EMOJI_MODIFIER_MASK != 0
}
#[inline(always)]
pub fn is_emoji_modifier_base(self) -> bool {
self.flags & Self::EMOJI_MODIFIER_BASE_MASK != 0
}
#[inline(always)]
pub fn is_default_ignorable(self) -> bool {
self.flags & Self::DEFAULT_IGNORABLE_MASK != 0
}
#[inline(always)]
pub fn contributes_to_shaping(self) -> bool {
self.flags & Self::CONTRIBUTES_TO_SHAPING_MASK != 0
}
#[inline(always)]
pub fn force_normalize(self) -> bool {
self.flags & Self::FORCE_NORMALIZE_MASK != 0
}
#[inline(always)]
pub fn is_grapheme_start(self) -> bool {
self.flags & Self::GRAPHEME_START_MASK != 0
}
#[inline(always)]
pub fn is_word_boundary(self) -> bool {
self.flags & Self::WORD_BOUNDARY_MASK != 0
}
#[inline(always)]
pub fn is_soft_wrap_opportunity(self) -> bool {
self.flags & Self::SOFT_WRAP_OPPORTUNITY_MASK != 0
}
#[inline(always)]
pub fn is_inline_object(self) -> bool {
self.flags & Self::INLINE_OBJECT_MASK != 0
}
}
pub(crate) fn analyze_text(
analyzer: &mut Analyzer,
text: &str,
options: &AnalysisOptions<'_>,
analysis: &mut Analysis,
data_sources: &AnalysisDataSources<'_>,
) {
struct DenseLineBreaks<'a> {
line_break: &'a [(Range<usize>, LineBreakConfig)],
next: usize,
cursor: usize,
text_len: usize,
}
impl<'a> DenseLineBreaks<'a> {
fn new(line_break: &'a [(Range<usize>, LineBreakConfig)], text_len: usize) -> Self {
Self {
line_break,
next: 0,
cursor: 0,
text_len,
}
}
}
impl Iterator for DenseLineBreaks<'_> {
type Item = (Range<usize>, SegmenterKey);
fn next(&mut self) -> Option<Self::Item> {
if self.cursor >= self.text_len {
return None;
}
while self
.line_break
.get(self.next)
.is_some_and(|(range, _)| range.is_empty())
{
self.next += 1;
}
match self.line_break.get(self.next) {
Some((range, _)) if self.cursor < range.start => {
let segment = self.cursor..range.start;
self.cursor = range.start;
Some((segment, SegmenterKey::DEFAULT))
}
Some((range, config)) => {
self.cursor = range.end;
self.next += 1;
Some((range.start..range.end, SegmenterKey::new(config)))
}
None => {
let segment = self.cursor..self.text_len;
self.cursor = self.text_len;
Some((segment, SegmenterKey::DEFAULT))
}
}
}
}
struct LineBreakSegmentIter<'a, I: Iterator> {
text: &'a str,
segments: I,
char_indices: core::str::CharIndices<'a>,
current_char: (usize, char),
building_range_start: usize,
previous_key: SegmenterKey,
done: bool,
}
impl<'a, I> LineBreakSegmentIter<'a, I>
where
I: Iterator<Item = (Range<usize>, SegmenterKey)>,
{
fn new(text: &'a str, segments: I, first_segment: (Range<usize>, SegmenterKey)) -> Self {
let mut char_indices = text.char_indices();
let current_char_len = char_indices.next().unwrap();
Self {
text,
segments,
char_indices,
current_char: current_char_len,
building_range_start: first_segment.0.start,
previous_key: first_segment.1,
done: false,
}
}
}
impl<'a, I> Iterator for LineBreakSegmentIter<'a, I>
where
I: Iterator<Item = (Range<usize>, SegmenterKey)>,
{
type Item = (&'a str, SegmenterKey, bool);
fn next(&mut self) -> Option<Self::Item> {
if self.done {
return None;
}
for (range, key) in self.segments.by_ref() {
assert!(range.start < range.end, "Segments must not be empty");
let style_start_index = range.start;
let mut prev_char_index = self.current_char;
while self.current_char.0 < style_start_index {
prev_char_index = self.current_char;
self.current_char = self.char_indices.next().unwrap();
}
let current_key = key;
if self.previous_key == current_key {
continue;
}
let prev_size = prev_char_index.1.len_utf8();
let size = self.current_char.1.len_utf8();
let substring = &self.text[self.building_range_start..style_start_index + size];
let result_style = self.previous_key;
self.building_range_start = style_start_index - prev_size;
self.previous_key = current_key;
return Some((substring, result_style, false));
}
self.done = true;
let last_substring = &self.text[self.building_range_start..self.text.len()];
Some((last_substring, self.previous_key, true))
}
}
if text.is_empty() {
analyzer
.bidi
.resolve(core::iter::empty(), options.base_direction);
analysis.paragraph_level = analyzer.bidi.base_level();
return;
}
let mut segments = DenseLineBreaks::new(options.line_break, text.len());
let first_segment = segments.next().unwrap();
let contiguous_substrings = LineBreakSegmentIter::new(text, segments, first_segment);
let mut global_offset = 0;
let line_boundary_positions = &mut analyzer.line_boundary_positions;
line_boundary_positions.clear();
for (substring_index, (substring, segmenter_key, last)) in contiguous_substrings.enumerate() {
let line_segmenter = data_sources.line_segmenter(segmenter_key);
let segmenter = line_segmenter.as_borrowed();
let line_boundaries_iter = segmenter.segment_str(substring);
let mut substring_chars = substring.chars();
if substring_index != 0 {
global_offset -= substring_chars.next().unwrap().len_utf8();
}
let last_len = substring_chars.next_back().unwrap().len_utf8();
for (index, pos) in line_boundaries_iter.enumerate() {
if index == 0 || pos == substring.len() {
continue;
}
if !last && pos == substring.len() - last_len {
continue;
}
line_boundary_positions.push(pos + global_offset);
}
if !last {
global_offset += substring.len() - last_len;
}
}
let word_segmenter = data_sources.word_segmenter();
let word_segmenter = word_segmenter.as_borrowed();
let mut wb_iter = word_segmenter.segment_str(text).peekable();
let mut gb_iter = data_sources
.grapheme_segmenter()
.segment_str(text)
.peekable();
let mut lb_iter = line_boundary_positions.iter().copied().peekable();
let mut prev_char = None;
let mut prev_prev_char = None;
let mut prev_is_mandatory_linebreak = false;
let mut prev_is_break_space = false;
let mut break_spaces_iter = options.break_spaces.iter().peekable();
let mut break_anywhere_iter = options
.line_break
.iter()
.filter(|(_, config)| config.line_break == LineBreak::Anywhere)
.map(|(range, _)| range)
.peekable();
let boundary_iter = text.char_indices().map(|(byte_pos, ch)| {
let mut is_word = wb_iter.next_if_eq(&byte_pos).is_some();
let is_grapheme_start = gb_iter.next_if_eq(&byte_pos).is_some();
let is_icu_line = lb_iter.next_if_eq(&byte_pos).is_some();
let properties = data_sources.properties(ch);
if !is_word
&& is_grapheme_start
&& properties.needs_dictionary_word_break()
&& !data_sources.has_word_dictionary(properties.script())
{
is_word = true;
}
while break_spaces_iter
.peek()
.is_some_and(|range| range.end <= byte_pos)
{
_ = break_spaces_iter.next();
}
let after_break_space = prev_is_break_space;
prev_is_break_space = break_spaces_iter
.peek()
.is_some_and(|range| range.contains(&byte_pos))
&& matches!(ch, ' ' | '\t' | '\u{3000}');
while break_anywhere_iter
.peek()
.is_some_and(|range| range.end <= byte_pos)
{
_ = break_anywhere_iter.next();
}
let is_break_anywhere = break_anywhere_iter
.peek()
.is_some_and(|range| range.contains(&byte_pos));
let is_mandatory_linebreak = properties.is_mandatory_linebreak();
let is_soft_wrap_opportunity = if is_mandatory_linebreak || prev_is_mandatory_linebreak {
false
} else {
let mut opportunity = if is_break_anywhere {
prev_char.is_some() && is_grapheme_start
} else {
is_icu_line || after_break_space
};
if let (Some(prev), Some(lb_override)) = (prev_char, options.line_break_override) {
let forced = lb_override(LineBreakContext {
before_before: prev_prev_char,
before: prev,
after: ch,
});
if let Some(forced) = forced {
opportunity = forced;
}
}
opportunity
};
prev_prev_char = prev_char;
prev_char = Some(ch);
prev_is_mandatory_linebreak = is_mandatory_linebreak;
(
is_soft_wrap_opportunity,
is_word,
is_grapheme_start,
ch,
properties,
)
});
let mut needs_bidi_resolution = false;
analysis.info.reserve(text.len());
for (is_soft_wrap_opportunity, is_word, is_grapheme_start, ch, properties) in boundary_iter {
let script = properties.script();
let grapheme_cluster_break = properties.grapheme_cluster_break();
let bidi_class = properties.bidi_class();
let general_category = properties.general_category();
let is_emoji_or_pictograph = properties.is_emoji_or_pictograph();
let is_variation_selector = properties.is_variation_selector();
let is_region_indicator = properties.is_regional_indicator();
let force_normalize = {
if matches!(grapheme_cluster_break, GraphemeClusterBreak::Extend) &&
ch as u32 != 0x200C && !is_variation_selector
{
true
} else {
matches!(grapheme_cluster_break, GraphemeClusterBreak::SpacingMark)
}
};
needs_bidi_resolution |= bidi::needs_bidi_resolution(bidi_class);
analysis.info.push(CharInfo::new(
script,
is_variation_selector,
is_region_indicator,
general_category == GeneralCategory::Control,
is_emoji_or_pictograph,
contributes_to_shaping(general_category, script),
force_normalize,
is_grapheme_start,
is_word,
is_soft_wrap_opportunity,
properties.is_emoji(),
properties.is_emoji_presentation(),
properties.is_emoji_modifier(),
properties.is_emoji_modifier_base(),
properties.is_default_ignorable(),
));
}
if !options.inline_objects.is_empty() {
mark_inline_objects(text, options, data_sources, &mut analysis.info);
}
if needs_bidi_resolution || options.base_direction == BaseDirection::Rtl {
analyzer.bidi.resolve(
text.chars().map(|ch| {
let bidi_class = data_sources.properties(ch).bidi_class();
let bracket = data_sources.brackets().get(ch);
(ch, (bidi_class, bracket))
}),
options.base_direction,
);
core::mem::swap(&mut analysis.levels, &mut analyzer.bidi.levels);
analysis.paragraph_level = analyzer.bidi.base_level();
}
}
#[inline(never)]
fn mark_inline_objects(
text: &str,
options: &AnalysisOptions<'_>,
data_sources: &AnalysisDataSources<'_>,
info: &mut [CharInfo],
) {
const LEN: usize = '\u{FFFC}'.len_utf8();
let is_mandatory = |ch| data_sources.properties(ch).is_mandatory_linebreak();
let combine = |a: Option<bool>, b: Option<bool>| match (a, b) {
(Some(false), _) | (_, Some(false)) => Some(false),
_ => a.or(b),
};
let mut char_index = 0;
let mut counted_to = 0;
for (i, object) in options.inline_objects.iter().enumerate() {
let byte = object.text_byte_start;
debug_assert!(
text[byte..].starts_with('\u{FFFC}'),
"An inline object must be a U+FFFC OBJECT REPLACEMENT CHARACTER, found none at byte {byte}"
);
char_index += text[counted_to..byte].chars().count();
counted_to = byte;
info[char_index].flags = info[char_index].flags & !CharInfo::CONTRIBUTES_TO_SHAPING_MASK
| CharInfo::INLINE_OBJECT_MASK;
let break_before = match options.inline_objects[..i].last() {
Some(previous) if previous.text_byte_start + LEN == byte => {
combine(previous.break_after, object.break_before)
}
_ => object.break_before,
};
let break_after = match options.inline_objects.get(i + 1) {
Some(next) if byte + LEN == next.text_byte_start => {
combine(object.break_after, next.break_before)
}
_ => object.break_after,
};
let mut before = text[..byte].chars().rev();
let (before, before_before) = (before.next(), before.next());
let after = text[byte + LEN..].chars().next();
for (char_index, byte, [before_before, before, after], condition) in [
(
char_index,
byte,
[before_before, before, Some('\u{FFFC}')],
break_before,
),
(
char_index + 1,
byte + LEN,
[before, Some('\u{FFFC}'), after],
break_after,
),
] {
let Some(info) = info.get_mut(char_index) else {
continue;
};
info.flags |= CharInfo::GRAPHEME_START_MASK;
let (Some(before), Some(after)) = (before, after) else {
continue;
};
if is_mandatory(before) || is_mandatory(after) {
continue;
}
let forced = options.line_break_override.and_then(|lb_override| {
lb_override(LineBreakContext {
before_before,
before,
after,
})
});
let is_break_anywhere = {
let i = options
.line_break
.partition_point(|(range, _)| range.end <= byte);
options.line_break.get(i).is_some_and(|(range, config)| {
range.contains(&byte) && config.line_break == LineBreak::Anywhere
})
};
let opportunity = forced
.or(condition)
.unwrap_or_else(|| info.is_soft_wrap_opportunity() || is_break_anywhere);
info.flags = info.flags & !CharInfo::SOFT_WRAP_OPPORTUNITY_MASK
| (opportunity as u16) << CharInfo::SOFT_WRAP_OPPORTUNITY_SHIFT;
}
}
}
#[inline(always)]
pub(crate) fn contributes_to_shaping(general_category: GeneralCategory, script: Script) -> bool {
if matches!(
general_category,
GeneralCategory::Control
| GeneralCategory::LineSeparator
| GeneralCategory::ParagraphSeparator
) {
return false;
}
!(general_category == GeneralCategory::Format && script != Script::Inherited)
}
#[cfg(test)]
mod test {
use alloc::vec::Vec;
use icu_segmenter::WordSegmenter;
use icu_segmenter::options::WordBreakInvariantOptions;
#[test]
fn icu4x_word_segmentation_patchup_required() {
let segmenter =
WordSegmenter::new_for_non_complex_scripts(WordBreakInvariantOptions::default());
let breaks: Vec<_> = segmenter.segment_str("中文文本测试").collect();
assert_eq!(breaks, &[0, 18]);
}
}