dxpdf 0.5.1

//! UAX #14: where may a line break?
//!
//! The companion to [`crate::render::spacing`], which owns the neighbouring
//! question — *where may space go* (the UAX #29 grapheme cluster). Both exist
//! for the same reason: the answer is needed in more than one place, and a
//! second copy of the rule drifts from the first. That is not hypothetical
//! here. Before issue #130 this engine answered "may a line break?" twice —
//! `fragment::text::split_into_words` decided where to cut a fragment, and
//! `layout::line::fit_lines` re-derived the answer by sniffing the fragment's
//! last character — against two lists that had already diverged: the cutter
//! broke after U+2012 FIGURE DASH, the fitter did not recognise it, so a
//! figure dash cut a fragment that was then not a break point.
//!
//! # What the old rule got wrong
//!
//! It knew four break characters: space, tab, hyphen-minus and four dashes.
//! Every consequence of that follows from what it could not see:
//!
//! * **Thai, Lao, Khmer and Burmese** put no spaces between words, so nothing
//!   in a paragraph of them was a break opportunity. They still wrapped —
//!   [`crate::render::layout::fragment::split_oversized_fragments`] cuts any
//!   over-wide fragment into grapheme clusters as a last resort — but the
//!   breaks landed inside words: `แบบจำ|ลอง` splits จำลอง. UAX #14 assigns these
//!   scripts class SA and hands them to "complex context analysis", which is
//!   what the LSTM models baked into `data/icu_data.blob` perform.
//! * **CJK** wrapped by the same accident, and so broke the two rules a
//!   Japanese reader notices first: [LB13] forbids a break *before* closing
//!   punctuation and [LB14] one *after* opening punctuation. A 127-line
//!   sample produced three lines beginning with `。`, one of them a line
//!   containing nothing else. CJK needs no dictionary — UAX #14's own rules
//!   break between ideographs (class ID) — only the rules.
//! * **Latin** was mostly right and wrong in three places, all of which this
//!   module changes: `ID-001` no longer breaks after the hyphen ([LB25]
//!   prohibits `HY NU`), a `/` and a URL path gain break opportunities, and
//!   an em dash becomes a unit of its own (class B2 breaks on both sides)
//!   rather than clinging to the word before it.
//!
//! # Tailoring
//!
//! [`break_offsets`] takes no options because the two OOXML properties that
//! would supply them are not consumed anywhere yet — see the note on this
//! module's `SEGMENTER` for which they are and what they map to. ICU4X's defaults
//! (`Strict` / `Normal`) are the right ones meanwhile: they are the plain
//! UAX #14 algorithm, which is what Word applies with East Asian typography
//! rules left on.
//!
//! [LB13]: https://www.unicode.org/reports/tr14/#LB13
//! [LB14]: https://www.unicode.org/reports/tr14/#LB14
//! [LB25]: https://www.unicode.org/reports/tr14/#LB25

use icu_segmenter::LineSegmenter;

thread_local! {
    /// Per-thread cache, for the same reason [`super::PROVIDER`] is one: the
    /// payloads a `LineSegmenter` holds are `Yoke`s over an `Rc` cart, so the
    /// type is `!Send`/`!Sync` and cannot live behind a process-wide `static`.
    /// Reuse matters more here than for a formatter — building one costs
    /// ~324 µs, since it deserializes the UAX #14 pair tables and all four
    /// complex-script LSTM models out of the blob.
    ///
    /// Built with ICU4X's default [`LineBreakOptions`] on purpose. Two OOXML
    /// properties tailor UAX #14 and neither is consumed by this engine yet:
    /// §17.3.1.45 `w:wordWrap` (parsed into
    /// `ParagraphProperties::word_wrap`, read by nothing) is
    /// `LineBreakWordOption::BreakAll`, and §17.3.1.16 `w:kinsoku` (not
    /// parsed at all) is `LineBreakStrictness`. Honouring either means the
    /// options stop being invariant per thread, so this becomes a small map
    /// keyed by the resolved options rather than a single value — a change
    /// confined to this file.
    ///
    /// [`LineBreakOptions`]: icu_segmenter::options::LineBreakOptions
    static SEGMENTER: LineSegmenter = super::with_data_provider(|provider| {
        LineSegmenter::try_new_auto_with_buffer_provider(provider, Default::default()).expect(
            "src/i18n/data/icu_data.blob is committed and regenerated by \
             scripts/make_icu_data.sh; a load failure here means its MARKERS \
             no longer cover icu_segmenter (SegmenterBreakLineV1, \
             SegmenterBreakGraphemeClusterV1, SegmenterLstmAutoV1)",
        )
    });
}

/// The byte offsets in `text` at which UAX #14 permits a line break, ascending.
///
/// Each offset is the index of the first byte *after* the break, so
/// `text[..offsets[0]]`, `text[offsets[0]..offsets[1]]`, … are the pieces a
/// line fitter may distribute across lines. The final offset is always
/// `text.len()`, and 0 is never present — ICU4X reports a boundary at the
/// start of the input for symmetry with its other segmenters, and its own
/// documentation calls that one "not a meaningful line break opportunity".
/// Empty input yields no offsets at all.
///
/// Every offset is a `char` boundary, and — because UAX #14 resolves class SA
/// through a grapheme-cluster segmenter — also a grapheme cluster boundary, so
/// a break can never separate a combining mark from its base.
///
/// The result is *mandatory and optional breaks together*; ICU4X does not
/// distinguish them and this engine has no use for the difference. Nothing
/// that would be a mandatory break (UAX #14 LB4/LB5) can reach this function:
/// CR, LF and every other C0 control except TAB are stripped upstream, and an
/// authored `<w:br/>` (§17.3.3.1) is a `Fragment::LineBreak` that never
/// becomes text.
pub fn break_offsets(text: &str) -> Vec<usize> {
    if text.is_empty() {
        return Vec::new();
    }
    SEGMENTER.with(|segmenter| segmenter.as_borrowed().segment_str(text).skip(1).collect())
}

#[cfg(test)]
mod tests {
    use super::*;

    /// The pieces `break_offsets` carves `text` into — what a caller actually
    /// builds from the offsets, and far easier to read in a failure message.
    fn pieces(text: &str) -> Vec<&str> {
        let mut out = Vec::new();
        let mut prev = 0;
        for offset in break_offsets(text) {
            out.push(&text[prev..offset]);
            prev = offset;
        }
        out
    }

    // ── The gap #130 exists to close ────────────────────────────────────────

    /// Class SA: Thai has no spaces, so the old rule found no break at all and
    /// the fallback cut mid-word. Expected pieces are real Thai words —
    /// "ภาษา" (language), "ไทย" (Thai), "เป็น" (is) — proving the LSTM model
    /// in the blob is loaded and consulted, not just that *some* break exists.
    #[test]
    fn thai_breaks_between_words() {
        assert_eq!(
            pieces("ภาษาไทยเป็นภาษาที่ไม่มีการเว้นวรรค"),
            [
                "ภาษา",
                "ไทย",
                "เป็น",
                "ภาษา",
                "ที่",
                "ไม่",
                "มี",
                "การ",
                "เว้นวรรค"
            ],
        );
    }

    /// The other three class-SA scripts the blob bakes models for. Asserting
    /// only "more than one piece" here: unlike Thai, nothing in this repo can
    /// review whether the Khmer/Lao/Burmese word divisions are *right*, and an
    /// expectation nobody can check is worse than one that states its own
    /// limit. Without the models these are one piece each.
    #[test]
    fn the_other_complex_scripts_have_models_too() {
        for (script, text) in [
            ("Khmer", "ក្រុមហ៊ុនតូចមួយកំពុងធ្វើការ"),
            ("Lao", "ພາສາລາວບໍ່ມີການເວັ້ນວັກລະຫວ່າງຄໍາ"),
            ("Burmese", "မြန်မာဘာသာစကားသည်စကားလုံးများကိုကွာဟမှုမရှိဘဲရေးသားသည်"),
        ] {
            let n = pieces(text).len();
            assert!(
                n > 1,
                "{script}: no word boundaries found, got {n} piece(s)"
            );
        }
    }

    /// [LB13]/[LB14], the two kinsoku rules whose absence put `。` at the start
    /// of 3 of 127 laid-out lines. A closing mark must stay with the text
    /// before it and an opening mark with the text after it — everywhere else
    /// between ideographs, a break is allowed (class ID).
    ///
    /// [LB13]: https://www.unicode.org/reports/tr14/#LB13
    /// [LB14]: https://www.unicode.org/reports/tr14/#LB14
    #[test]
    fn japanese_keeps_punctuation_off_a_line_edge() {
        assert_eq!(
            pieces("日本語の文章。「引用」とか"),
            ["日", "本", "語", "の", "文", "章。", "「引", "用」", "と", "か"],
        );
    }

    // ── Latin: what stays, and the three things that change ─────────────────

    /// The vocabulary the committed corpus is actually made of. These are the
    /// cases that must not move, because they are what the pixel-diff over
    /// `test-cases/` is checking.
    #[test]
    fn ordinary_latin_text_breaks_exactly_where_it_used_to() {
        assert_eq!(
            pieces("The quick brown fox jumps over the lazy dog."),
            ["The ", "quick ", "brown ", "fox ", "jumps ", "over ", "the ", "lazy ", "dog."],
        );
        assert_eq!(pieces("Türöffner-Gerät"), ["Türöffner-", "Gerät"]);
        assert_eq!(pieces("10:30–12:00 Uhr"), ["10:30–", "12:00 ", "Uhr"]);
        assert_eq!(pieces("„Zitat“ und mehr"), ["„Zitat“ ", "und ", "mehr"]);
        assert_eq!(pieces("e.g. i.e. etc."), ["e.g. ", "i.e. ", "etc."]);
        assert_eq!(pieces("Datei_name.txt"), ["Datei_name.txt"]);
    }

    /// [LB25]: a hyphen followed by digits is inside a number, not between two
    /// words. The old rule broke `ID-001` after the hyphen; UAX #14 does not,
    /// which is the change most likely to show up in a pixel-diff of the
    /// German corpus (part numbers, `DIN VDE 0100-600`).
    ///
    /// [LB25]: https://www.unicode.org/reports/tr14/#LB25
    #[test]
    fn a_hyphen_before_digits_is_not_a_break() {
        assert_eq!(pieces("ID-001"), ["ID-001"]);
        assert_eq!(pieces("DIN VDE 0100-600"), ["DIN ", "VDE ", "0100-600"]);
        // …but between words it still is, exactly as before.
        assert_eq!(pieces("Anlagen-freigabe"), ["Anlagen-", "freigabe"]);
    }

    /// U+2011 NON-BREAKING HYPHEN (class GL) must hold a token together —
    /// the property the old `non_breaking_hyphen_stays_inside_word` test
    /// covered, restated at the layer that now decides it.
    #[test]
    fn a_non_breaking_hyphen_holds_a_token_together() {
        assert_eq!(pieces("ID\u{2011}001"), ["ID\u{2011}001"]);
    }

    /// The old cutter broke after U+2012 FIGURE DASH and the old fitter did
    /// not recognise it, so the resulting fragment boundary was unusable. One
    /// rule, one answer: it is a break, and the caller reading this offset is
    /// the same caller that acts on it.
    #[test]
    fn a_figure_dash_is_a_break_opportunity() {
        assert_eq!(pieces("co\u{2012}operate"), ["co\u{2012}", "operate"]);
    }

    /// Two genuine behaviour changes on Latin text, locked in so they are
    /// decisions rather than surprises. A slash gains a break opportunity
    /// after it, which is what stops a long URL from overflowing; an em dash
    /// (class B2) becomes its own unit and may therefore begin a line, which
    /// is UAX #14-correct and the one change that reads as a regression
    /// against Word.
    #[test]
    fn slashes_and_em_dashes_gain_opportunities() {
        assert_eq!(pieces("a/b/c"), ["a/", "b/", "c"]);
        assert_eq!(
            pieces("https://example.com/some/path"),
            ["https://", "example.com/", "some/", "path"],
        );
        assert_eq!(pieces("long—dash—text"), ["long", "—", "dash", "—", "text"],);
    }

    /// TAB survives fragment building (XML §2.1 strips every C0 control but
    /// this one), so it reaches the segmenter; class BA breaks after it, as
    /// the old rule did.
    #[test]
    fn a_tab_breaks_after_itself() {
        assert_eq!(pieces("a\tb"), ["a\t", "b"]);
    }

    // ── Contract ────────────────────────────────────────────────────────────

    #[test]
    fn empty_text_has_no_offsets() {
        assert!(break_offsets("").is_empty());
    }

    /// The three properties every caller relies on: 0 is dropped, the end is
    /// always present, and the offsets are usable as slice indices.
    #[test]
    fn offsets_are_ascending_char_boundaries_ending_at_the_length() {
        for text in [
            "hello world",
            "ภาษาไทยเป็นภาษา",
            "日本語の文章。",
            "e\u{301}x—y",
            "x",
        ] {
            let offsets = break_offsets(text);
            assert_eq!(offsets.last(), Some(&text.len()), "{text:?}");
            assert!(!offsets.contains(&0), "{text:?} must not report offset 0");
            assert!(
                offsets.windows(2).all(|w| w[0] < w[1]),
                "{text:?} offsets must ascend: {offsets:?}",
            );
            for &o in &offsets {
                assert!(text.is_char_boundary(o), "{text:?} offset {o}");
            }
        }
    }

    /// A break must never separate a combining mark from its base — the same
    /// invariant `crate::render::spacing` protects for the neighbouring
    /// question, here inherited from UAX #14 rather than restated.
    #[test]
    fn a_combining_mark_is_never_cut_from_its_base() {
        assert_eq!(pieces("e\u{301} x"), ["e\u{301} ", "x"]);
        assert_eq!(pieces("e\u{301}"), ["e\u{301}"]);
    }
}