pdfrum_text/
word.rs

1//! The words of a page, as a view over the character list.
2//!
3//! A word here is what an extraction or citation pipeline means by one: a
4//! run of the reading-order text between two whitespace characters, with the
5//! box its glyphs cover and the font the first glyph was set in. Nothing is
6//! laid out again — every field is read off the [`TextPage`] the extractor
7//! already built, so the split costs one pass over `chars`.
8//!
9//! This is **not** the scripting API's word list ([`crate::words`]), which
10//! walks the content stream as written and splits on a different rule.
11
12use crate::TextPage;
13use crate::charinfo::CharBox;
14use crate::index::CharIndex;
15use kurbo::Rect;
16use std::ops::Range;
17
18/// One word of a page's text, from [`TextPage::words`].
19#[derive(Debug, Clone, PartialEq)]
20pub struct Word {
21    /// The word, without surrounding whitespace.
22    pub text: String,
23    /// The union of its characters' boxes, in page space — the same
24    /// coordinates [`TextPage::rects`] reports. A word none of whose glyphs
25    /// has a visible box reports [`Rect::ZERO`].
26    pub rect: Rect,
27    /// The characters this word was cut from, so [`TextPage::slice`] on it
28    /// gives back [`text`](Self::text) and the characters around it give
29    /// context.
30    pub range: Range<CharIndex>,
31    /// The base font name of the first character's font, when the font has
32    /// one — see [`TextPage::font_name`].
33    pub font: Option<String>,
34    /// The first character's font size in page points: the `Tf` operand
35    /// scaled by the text matrix, so `1 Tf` under a `12 0 0 12` matrix reads
36    /// as 12.
37    pub size: f64,
38}
39
40/// Whether a character ends a word: the extractor invented it (a space or a
41/// line break the geometry implied), or it is Unicode whitespace.
42fn is_separator(info: &CharBox) -> bool {
43    info.is_generated() || char::from_u32(info.unicode).is_some_and(char::is_whitespace)
44}
45
46/// The visible box of one character, normalized, or `None` where
47/// [`TextPage::rects`] would skip it.
48fn visible_box(info: &CharBox) -> Option<Rect> {
49    if info.is_generated() {
50        return None;
51    }
52    let rect = info.char_box;
53    if rect.width().abs() < 0.01 || rect.height().abs() < 0.01 {
54        return None;
55    }
56    Some(Rect::new(
57        rect.x0.min(rect.x1),
58        rect.y0.min(rect.y1),
59        rect.x0.max(rect.x1),
60        rect.y0.max(rect.y1),
61    ))
62}
63
64/// The font size a character was drawn at, in page points.
65fn point_size(info: &CharBox) -> f64 {
66    let [_, _, c, d, ..] = info.matrix.as_coeffs();
67    f64::from(info.font_size).abs() * c.hypot(d)
68}
69
70impl TextPage {
71    /// The page's words in reading order: the runs of
72    /// [`chars`](Self::chars) between whitespace, each with the box its
73    /// glyphs cover and the font of its first character.
74    ///
75    /// A word's [`text`](Word::text) is what [`slice`](Self::slice) gives for
76    /// its [`range`](Word::range), so a hyphenated line break shows up as
77    /// two words, the first ending in `U+00AD`. A run whose characters all
78    /// vanish from the search text — the charcode-0 passthrough, the control
79    /// code points — is not a word.
80    ///
81    /// # Examples
82    ///
83    /// ```
84    /// # use pdfrum_text::TextPage;
85    /// let page = TextPage::default();
86    /// assert!(page.words().is_empty());
87    /// ```
88    #[must_use]
89    pub fn words(&self) -> Vec<Word> {
90        let mut out = Vec::new();
91        let mut start: Option<usize> = None;
92        for (position, info) in self.chars.iter().enumerate() {
93            match (start, is_separator(info)) {
94                (None, false) => start = Some(position),
95                (Some(from), true) => {
96                    out.extend(self.word(from..position));
97                    start = None;
98                }
99                (None, true) | (Some(_), false) => {}
100            }
101        }
102        if let Some(from) = start {
103            out.extend(self.word(from..self.chars.len()));
104        }
105        out
106    }
107
108    /// One word over a run of consecutive non-separator characters, or
109    /// `None` when the run has no text.
110    fn word(&self, positions: Range<usize>) -> Option<Word> {
111        let range = CharIndex::new(positions.start)..CharIndex::new(positions.end);
112        let text = self.slice(range.clone());
113        let text = text.trim();
114        if text.is_empty() {
115            return None;
116        }
117        let chars = self.chars.get(positions).unwrap_or_default();
118        let first = chars.first()?;
119        let rect = chars
120            .iter()
121            .filter_map(visible_box)
122            .reduce(|a, b| a.union(b))
123            .unwrap_or(Rect::ZERO);
124        let font = self.font_name(range.start).map(str::to_owned);
125        Some(Word {
126            text: text.to_owned(),
127            rect,
128            range,
129            font,
130            size: point_size(first),
131        })
132    }
133}
134
135#[cfg(test)]
136mod tests {
137    #![allow(
138        clippy::float_cmp,
139        clippy::indexing_slicing,
140        reason = "test fixtures pin exact values"
141    )]
142
143    use super::*;
144    use crate::charinfo::{CharType, ObjectIndex};
145    use crate::index;
146    use kurbo::{Affine, Point};
147    use pdfrum_font::CharCode;
148
149    /// A page whose characters are `text`, each one glyph wide at `x = 10 *
150    /// position`, drawn by object `0` unless the character is a space — a
151    /// space is a generated one, as the extractor would emit for an inter-word
152    /// gap.
153    fn page(text: &str, matrix: Affine) -> TextPage {
154        let chars: Vec<CharBox> = text
155            .chars()
156            .enumerate()
157            .map(|(position, ch)| {
158                let generated = ch == ' ';
159                #[expect(clippy::cast_precision_loss, reason = "a test index is tiny")]
160                let x = position as f64 * 10.0;
161                let char_box = if generated {
162                    Rect::ZERO
163                } else {
164                    Rect::new(x, 0.0, x + 8.0, 10.0)
165                };
166                CharBox {
167                    char_type: if generated {
168                        CharType::Generated
169                    } else {
170                        CharType::Normal
171                    },
172                    unicode: u32::from(ch),
173                    code: (!generated).then_some(CharCode(u32::from(ch))),
174                    origin: Point::new(x, 0.0),
175                    char_box,
176                    loose_char_box: char_box,
177                    matrix,
178                    object: (!generated).then_some(ObjectIndex(0)),
179                    font_size: if generated { 1.0 } else { 12.0 },
180                    angle: 0.0,
181                }
182            })
183            .collect();
184        let runs = index::build(&chars);
185        let mut page = TextPage {
186            search_text: text.chars().collect(),
187            chars,
188            runs,
189            ..TextPage::default()
190        };
191        page.fonts.insert(ObjectIndex(0), "Helvetica".to_owned());
192        page
193    }
194
195    #[test]
196    fn words_split_on_whitespace_and_slice_back_to_themselves() {
197        let page = page("Hello, world!", Affine::IDENTITY);
198        let words = page.words();
199        let texts: Vec<&str> = words.iter().map(|w| w.text.as_str()).collect();
200        assert_eq!(texts, ["Hello,", "world!"]);
201        for word in &words {
202            assert_eq!(page.slice(word.range.clone()), word.text);
203        }
204        assert_eq!(words[0].range, CharIndex::new(0)..CharIndex::new(6));
205        assert_eq!(words[1].range, CharIndex::new(7)..CharIndex::new(13));
206    }
207
208    #[test]
209    fn a_word_covers_the_union_of_its_glyph_boxes() {
210        let page = page("ab cd", Affine::IDENTITY);
211        let words = page.words();
212        assert_eq!(words[0].rect, Rect::new(0.0, 0.0, 18.0, 10.0));
213        assert_eq!(words[1].rect, Rect::new(30.0, 0.0, 48.0, 10.0));
214    }
215
216    #[test]
217    fn the_font_and_size_come_from_the_first_character() {
218        let page = page("ab", Affine::scale(2.0));
219        let words = page.words();
220        assert_eq!(words[0].font.as_deref(), Some("Helvetica"));
221        // `12 Tf` under a doubling matrix is 24 points on the page.
222        assert_eq!(words[0].size, 24.0);
223    }
224
225    #[test]
226    fn leading_and_trailing_whitespace_makes_no_word() {
227        let page = page("  one  ", Affine::IDENTITY);
228        let words = page.words();
229        assert_eq!(words.len(), 1);
230        assert_eq!(words[0].text, "one");
231        assert_eq!(words[0].range, CharIndex::new(2)..CharIndex::new(5));
232    }
233
234    #[test]
235    fn a_run_the_search_text_drops_is_not_a_word() {
236        // `U+0003` is a control code point `search_text` drops.
237        let mut page = page("a \u{3} b", Affine::IDENTITY);
238        page.search_text = "a  b".chars().collect();
239        page.runs = index::build(&page.chars);
240        let texts: Vec<String> = page.words().into_iter().map(|w| w.text).collect();
241        assert_eq!(texts, ["a", "b"]);
242    }
243
244    #[test]
245    fn font_name_reads_through_the_character_to_its_object() {
246        let page = page("a b", Affine::IDENTITY);
247        assert_eq!(page.font_name(CharIndex::new(0)), Some("Helvetica"));
248        // The generated space has no object.
249        assert_eq!(page.font_name(CharIndex::new(1)), None);
250        assert_eq!(page.font_name(CharIndex::new(9)), None);
251    }
252}
pdfrum_text/word.rs

pdfrum_text/
word.rs
