1use crate::blocks::{Block, ExtractOptions};
30use crate::text::TextSpan;
31
32#[derive(Debug, Clone, PartialEq)]
40pub struct Word {
41 pub page: usize,
43 pub text: String,
45 pub bbox: [f64; 4],
49 pub block: usize,
56 pub line: usize,
58 pub word: usize,
60 pub dir: f64,
65 pub bold: bool,
67 pub italic: bool,
69 pub font_size: f64,
71}
72
73fn union(a: [f64; 4], b: [f64; 4]) -> [f64; 4] {
74 [
75 a[0].min(b[0]),
76 a[1].min(b[1]),
77 a[2].max(b[2]),
78 a[3].max(b[3]),
79 ]
80}
81
82fn span_box(span: &TextSpan, chars: usize) -> [f64; 4] {
89 let width = if span.width > 0.0 {
90 span.width
91 } else {
92 span.font_size.max(1.0) * 0.5 * chars as f64
93 };
94 let height = if span.height > 0.0 {
95 span.height
96 } else {
97 span.font_size.max(1.0)
98 };
99 [span.x, span.y, span.x + width, span.y + height]
100}
101
102fn span_angle(span: &TextSpan) -> (f64, (f64, f64)) {
110 let Some(coeffs) = span.transform.as_ref() else {
111 return (0.0, (1.0, 0.0));
112 };
113 let (a, b) = (coeffs[0], coeffs[1]);
114 let length = (a * a + b * b).sqrt();
115 if length <= f64::EPSILON {
116 return (0.0, (1.0, 0.0));
117 }
118 let dir = b.atan2(a).to_degrees();
119 (if dir == 0.0 { 0.0 } else { dir }, (a / length, b / length))
120}
121
122fn turned_box(bound: [f64; 4], direction: (f64, f64)) -> [f64; 4] {
130 let (dx, dy) = direction;
131 if dx == 1.0 && dy == 0.0 {
132 return bound;
133 }
134 let (x0, y0) = (bound[0], bound[1]);
135 let (advance, size) = (bound[2] - bound[0], bound[3] - bound[1]);
136 let mut envelope: Option<[f64; 4]> = None;
137 for (u, v) in [(0.0, 0.0), (advance, 0.0), (advance, size), (0.0, size)] {
138 let corner = [x0 + u * dx - v * dy, y0 + u * dy + v * dx];
139 envelope = Some(match envelope {
140 Some(b) => [
141 b[0].min(corner[0]),
142 b[1].min(corner[1]),
143 b[2].max(corner[0]),
144 b[3].max(corner[1]),
145 ],
146 None => [corner[0], corner[1], corner[0], corner[1]],
147 });
148 }
149 envelope.unwrap_or(bound)
150}
151
152fn along(x: f64, y: f64, direction: (f64, f64)) -> f64 {
155 x * direction.0 + y * direction.1
156}
157
158fn gap_between(left: [f64; 4], right: [f64; 4], direction: (f64, f64)) -> f64 {
164 along(right[0], right[1], direction)
165 - (along(left[0], left[1], direction) + (left[2] - left[0]))
166}
167
168fn skipped_glyph_word(
179 chars: &[char],
180 word_of_char: &[Option<usize>],
181 glyphs: &[[f64; 4]],
182 index: usize,
183 owner: usize,
184 direction: (f64, f64),
185 em: f64,
186) -> Option<usize> {
187 if em <= 0.0 || em.is_nan() || chars.len() <= glyphs.len() {
190 return None;
191 }
192 if owner > 0 && chars[owner - 1].is_whitespace() {
193 return None;
194 }
195 if owner + 1 < chars.len() && chars[owner + 1].is_whitespace() {
196 return None;
197 }
198 if index == 0 || index + 1 >= glyphs.len() {
199 return None;
200 }
201 let joins_left = crate::words_join::pieces_join(
202 gap_between(glyphs[index - 1], glyphs[index], direction),
203 0.0,
204 em,
205 );
206 let breaks_right = !crate::words_join::pieces_join(
207 gap_between(glyphs[index], glyphs[index + 1], direction),
208 0.0,
209 em,
210 );
211 if joins_left && breaks_right {
212 (0..owner).rev().find_map(|k| word_of_char[k])
213 } else {
214 None
215 }
216}
217
218fn words_of_span(span: &TextSpan) -> Vec<(String, [f64; 4])> {
237 let chars: Vec<char> = span.text.chars().collect();
238 let n = chars.len();
239 if n == 0 {
240 return Vec::new();
241 }
242 let glyphs = &span.char_bounds;
243 let g = glyphs.len();
244 let box_of_span = span_box(span, n);
245
246 let mut word_of_char: Vec<Option<usize>> = Vec::with_capacity(n);
248 let mut texts: Vec<String> = Vec::new();
249 let mut ranges: Vec<(usize, usize)> = Vec::new();
250 let mut in_word = false;
251 for (i, ch) in chars.iter().enumerate() {
252 if ch.is_whitespace() {
253 in_word = false;
254 word_of_char.push(None);
255 } else {
256 if !in_word {
257 texts.push(String::new());
258 ranges.push((i, i));
259 in_word = true;
260 }
261 let w = texts.len() - 1;
262 texts[w].push(*ch);
263 ranges[w].1 = i + 1;
264 word_of_char.push(Some(w));
265 }
266 }
267 if texts.is_empty() {
268 return Vec::new();
269 }
270
271 let (_, direction) = span_angle(span);
272 let mut boxes: Vec<Option<[f64; 4]>> = vec![None; texts.len()];
273 for (index, glyph_box) in glyphs.iter().enumerate() {
277 let owner = (0..n)
278 .find(|&c| c * g / n == index)
279 .or_else(|| (0..n).rev().find(|&c| c * g / n <= index));
280 let Some(c) = owner else { continue };
281 let word = (c..n)
285 .take_while(|&k| k * g / n == index || k == c)
286 .find_map(|k| word_of_char[k])
287 .or_else(|| {
288 skipped_glyph_word(
289 &chars,
290 &word_of_char,
291 glyphs,
292 index,
293 c,
294 direction,
295 span.font_size,
296 )
297 });
298 if let Some(w) = word {
299 let envelope = turned_box(*glyph_box, direction);
300 boxes[w] = Some(match boxes[w] {
301 Some(b) => union(b, envelope),
302 None => envelope,
303 });
304 }
305 }
306
307 let total = n as f64;
308 texts
309 .into_iter()
310 .zip(ranges)
311 .zip(boxes)
312 .map(|((text, (start, end)), bbox)| {
313 let bbox = bbox.unwrap_or_else(|| {
314 let left =
315 box_of_span[0] + (box_of_span[2] - box_of_span[0]) * start as f64 / total;
316 let right = if end == n {
317 box_of_span[2]
318 } else {
319 box_of_span[0] + (box_of_span[2] - box_of_span[0]) * end as f64 / total
320 };
321 [left, box_of_span[1], right, box_of_span[3]]
322 });
323 (text, bbox)
324 })
325 .collect()
326}
327
328fn starts_with_whitespace(text: &str) -> bool {
330 text.chars().next().is_some_and(|c| c.is_whitespace())
331}
332
333fn ends_with_whitespace(text: &str) -> bool {
335 text.chars().last().is_some_and(|c| c.is_whitespace())
336}
337
338struct Tail {
340 span_pos: usize,
342 y: f64,
344 font_size: f64,
346 style_len: usize,
348}
349
350fn words_of_line(
359 block: &Block,
360 block_index: usize,
361 line_index: usize,
362 line: &[usize],
363) -> Vec<Word> {
364 let mut out: Vec<Word> = Vec::new();
365 let mut tail: Option<Tail> = None;
366 let mut prev_ends_with_space = false;
367 for (pos, span_index) in line.iter().enumerate() {
368 let span = &block.spans[*span_index];
369 let (span_dir, _) = span_angle(span);
370 let starts_with_space = starts_with_whitespace(&span.text);
371 for (piece_index, (text, bbox)) in words_of_span(span).iter().enumerate() {
372 let join = piece_index == 0
373 && tail.as_ref().is_some_and(|t| {
374 let Some(last) = out.last() else {
375 return false;
376 };
377 if t.span_pos + 1 != pos || prev_ends_with_space || starts_with_space {
378 return false;
379 }
380 let em = t.font_size.max(span.font_size);
381 let gap = bbox[0] - last.bbox[2];
382 crate::words_join::pieces_join(gap, (t.y - span.y).abs(), em)
383 });
384 let len = text.chars().count();
385 if join {
386 let last = out.last_mut().expect("join implies a word");
387 last.text.push_str(text);
388 last.bbox = union(last.bbox, *bbox);
389 if let Some(t) = tail.as_mut() {
390 if len > t.style_len {
391 last.bold = span.is_bold;
392 last.italic = span.is_italic;
393 last.font_size = span.font_size;
394 last.dir = span_dir;
395 t.style_len = len;
396 }
397 t.span_pos = pos;
398 t.y = span.y;
399 t.font_size = span.font_size;
400 }
401 } else {
402 out.push(Word {
403 page: block.page,
404 text: text.clone(),
405 bbox: *bbox,
406 block: block_index,
407 line: line_index,
408 word: 0,
409 dir: span_dir,
410 bold: span.is_bold,
411 italic: span.is_italic,
412 font_size: span.font_size,
413 });
414 tail = Some(Tail {
415 span_pos: pos,
416 y: span.y,
417 font_size: span.font_size,
418 style_len: len,
419 });
420 }
421 }
422 prev_ends_with_space = ends_with_whitespace(&span.text);
423 }
424 for (index, word) in out.iter_mut().enumerate() {
425 word.word = index;
426 }
427 out
428}
429
430pub(crate) fn words_from_blocks(blocks: &[Block]) -> Vec<Word> {
436 let mut out = Vec::new();
437 for (block_index, block) in blocks.iter().enumerate() {
438 for (line_index, line) in crate::text::lines_of_block(block).iter().enumerate() {
439 out.extend(words_of_line(block, block_index, line_index, line));
440 }
441 }
442 out
443}
444
445impl ExtractOptions {
446 #[must_use]
449 pub fn with_word_boxes(mut self, word_boxes: bool) -> Self {
450 self.word_boxes = word_boxes;
451 self
452 }
453}
454
455pub fn to_json_with_words(blocks: &[Block]) -> String {
461 use crate::json::INDENT;
462 if blocks.is_empty() {
463 return format!("{{\n{INDENT}\"blocks\": []\n}}\n");
464 }
465 let pad = INDENT.repeat(2);
466 let inner = INDENT.repeat(3);
467 let written: Vec<String> = blocks
468 .iter()
469 .enumerate()
470 .map(|(index, block)| {
471 let plain = crate::json::block(block, 2);
472 let suffix = format!("\n{pad}}}");
473 let stem = plain.strip_suffix(&suffix).expect(
474 "to_json_with_words: the block object does not close as json::block writes it",
475 );
476 format!(
477 "{stem},\n{inner}\"lines\": {}{suffix}",
478 lines_json(block, index)
479 )
480 })
481 .collect();
482 format!(
483 "{{\n{INDENT}\"blocks\": [\n{}\n{INDENT}]\n}}\n",
484 written.join(",\n")
485 )
486}
487
488fn lines_json(block: &Block, block_index: usize) -> String {
495 use crate::json::{number, string, INDENT};
496 let inner = INDENT.repeat(3);
497 let pad = INDENT.repeat(4);
498 let mut lines = Vec::new();
499 for (line_index, line) in crate::text::lines_of_block(block).iter().enumerate() {
500 let words = words_of_line(block, block_index, line_index, line);
501 let spans = line
502 .iter()
503 .map(|i| i.to_string())
504 .collect::<Vec<_>>()
505 .join(", ");
506 let mut entries = Vec::with_capacity(words.len());
507 for word in &words {
508 let bbox = word
509 .bbox
510 .iter()
511 .map(|v| number(*v))
512 .collect::<Vec<_>>()
513 .join(", ");
514 entries.push(format!(
515 "{{\"text\": {}, \"bbox\": [{bbox}], \"word\": {}, \"dir\": {}}}",
516 string(&word.text),
517 word.word,
518 number(word.dir)
519 ));
520 }
521 lines.push(format!(
522 "{pad}{{\"spans\": [{spans}], \"words\": [{}]}}",
523 entries.join(", ")
524 ));
525 }
526 if lines.is_empty() {
527 "[]".to_string()
528 } else {
529 format!("[\n{}\n{inner}]", lines.join(",\n"))
530 }
531}
532
533pub fn to_words_json_lines(words: &[Word]) -> String {
538 use crate::json::{number, string};
539 let mut out = String::new();
540 for word in words {
541 let bbox = word
542 .bbox
543 .iter()
544 .map(|v| number(*v))
545 .collect::<Vec<_>>()
546 .join(", ");
547 out.push_str(&format!(
548 "{{\"page\": {}, \"block\": {}, \"line\": {}, \"word\": {}, \"text\": {}, \"bbox\": [{bbox}], \"dir\": {}}}\n",
549 word.page,
550 word.block,
551 word.line,
552 word.word,
553 string(&word.text),
554 number(word.dir)
555 ));
556 }
557 out
558}
559
560#[cfg(test)]
561mod tests {
562 use super::*;
563 use crate::blocks::BlockKind;
564
565 fn block_of(spans: Vec<TextSpan>) -> Block {
566 let mut block = Block::new(
567 BlockKind::Paragraph,
568 0,
569 [0.0, 0.0, 100.0, 100.0],
570 String::new(),
571 0,
572 );
573 block.spans = spans;
574 block
575 }
576
577 fn span_with(text: &str, x: f64, y: f64, glyph_width: f64, size: f64) -> TextSpan {
578 let mut span = TextSpan {
579 text: text.to_string(),
580 x,
581 y,
582 width: glyph_width * text.chars().count() as f64,
583 height: size,
584 font_size: size,
585 ..Default::default()
586 };
587 let mut cursor = x;
588 for _ in text.chars() {
589 span.char_bounds
590 .push([cursor, y, cursor + glyph_width, y + size]);
591 cursor += glyph_width;
592 }
593 span
594 }
595
596 fn turned_span(
602 text: &str,
603 x: f64,
604 y: f64,
605 glyph_width: f64,
606 size: f64,
607 degrees: f64,
608 ) -> TextSpan {
609 let mut span = span_with(text, x, y, glyph_width, size);
610 let (sin, cos) = degrees.to_radians().sin_cos();
611 span.transform = Some([cos, sin, -sin, cos, x, y]);
612 span
613 }
614
615 #[test]
616 fn a_word_takes_the_union_of_its_glyphs() {
617 let span = span_with("hello world", 10.0, 20.0, 6.0, 12.0);
618 let words = words_of_span(&span);
619 assert_eq!(words.len(), 2);
620 assert_eq!(words[0].0, "hello");
621 assert_eq!(words[1].0, "world");
622 assert_eq!(words[0].1, [10.0, 20.0, 40.0, 32.0]);
623 assert_eq!(words[1].1, [46.0, 20.0, 76.0, 32.0]);
624 }
625
626 #[test]
627 fn a_span_without_boxes_splits_proportionally() {
628 let span = TextSpan {
629 text: "ab cd".to_string(),
630 x: 0.0,
631 y: 0.0,
632 width: 100.0,
633 height: 10.0,
634 font_size: 10.0,
635 ..Default::default()
636 };
637 assert!(span.char_bounds.is_empty());
638 let words = words_of_span(&span);
639 assert_eq!(words.len(), 2);
640 assert_eq!(words[0].1, [0.0, 0.0, 40.0, 10.0]);
641 assert_eq!(words[1].1, [60.0, 0.0, 100.0, 10.0]);
642 }
643
644 #[test]
645 fn an_inserted_space_owns_no_glyph() {
646 let mut span = span_with("ab", 0.0, 0.0, 10.0, 10.0);
649 span.text = "a b".to_string();
650 span.width = 30.0;
651 let words = words_of_span(&span);
652 assert_eq!(words.len(), 2);
653 assert_eq!(words[0].1, [0.0, 0.0, 10.0, 10.0]);
654 assert_eq!(words[1].1, [10.0, 0.0, 20.0, 10.0]);
655 }
656
657 #[test]
658 fn two_touching_spans_join_into_one_word() {
659 let mut hel = span_with("Hel", 0.0, 20.0, 6.0, 12.0);
663 hel.is_bold = true;
664 let lo = span_with("lo", 18.0, 20.0, 6.0, 10.0);
665 let words = words_from_blocks(&[block_of(vec![hel, lo])]);
666 assert_eq!(words.len(), 1, "the split word did not join: {words:?}");
667 assert_eq!(words[0].text, "Hello");
668 assert_eq!(words[0].bbox, [0.0, 20.0, 30.0, 32.0]);
669 assert!(words[0].bold, "the longer piece lends the style");
670 assert_eq!(words[0].font_size, 12.0);
671 }
672
673 #[test]
674 fn two_spans_a_word_space_apart_stay_two_words() {
675 let hel = span_with("Hel", 0.0, 20.0, 6.0, 12.0);
678 let lo = span_with("lo", 18.0 + 0.4 * 12.0, 20.0, 6.0, 12.0);
679 let words = words_from_blocks(&[block_of(vec![hel, lo])]);
680 assert_eq!(words.len(), 2, "the space did not hold: {words:?}");
681 assert_eq!(words[0].text, "Hel");
682 assert_eq!(words[1].text, "lo");
683 assert_eq!(words[0].line, words[1].line, "not on one line");
684 }
685
686 #[test]
687 fn a_superscript_right_after_its_base_stays_its_own_word() {
688 let x = span_with("x", 0.0, 20.0, 6.0, 10.0);
691 let two = span_with("2", 6.0, 24.0, 6.0, 10.0);
692 let words = words_from_blocks(&[block_of(vec![x, two])]);
693 assert_eq!(words.len(), 2, "the superscript joined: {words:?}");
694 assert_eq!(words[0].text, "x");
695 assert_eq!(words[1].text, "2");
696 assert_eq!(words[0].line, words[1].line, "not on one line");
697 }
698
699 #[test]
700 fn a_tied_join_keeps_the_left_pieces_style() {
701 let mut ab = turned_span("ab", 0.0, 20.0, 6.0, 10.0, 10.0);
705 ab.is_bold = true;
706 let cd = span_with("cd", 12.0, 20.0, 6.0, 14.0);
707 let words = words_from_blocks(&[block_of(vec![ab, cd])]);
708 assert_eq!(words.len(), 1, "the split word did not join: {words:?}");
709 assert_eq!(words[0].text, "abcd");
710 assert!(words[0].bold, "the tie went right");
711 assert_eq!(words[0].font_size, 10.0);
712 assert!(
713 (words[0].dir - 10.0).abs() < 1e-9,
714 "the tie went right: {}",
715 words[0].dir
716 );
717 }
718
719 #[test]
720 fn a_join_takes_the_longer_pieces_angle() {
721 let hel = turned_span("Hel", 0.0, 20.0, 6.0, 12.0, 10.0);
725 let lo = span_with("lo", 18.0, 20.0, 6.0, 12.0);
726 let words = words_from_blocks(&[block_of(vec![hel, lo])]);
727 assert_eq!(words.len(), 1, "the split word did not join: {words:?}");
728 assert_eq!(words[0].text, "Hello");
729 assert!(
730 (words[0].dir - 10.0).abs() < 1e-9,
731 "the angle went right: {}",
732 words[0].dir
733 );
734 }
735
736 #[test]
737 fn a_join_takes_the_longer_pieces_angle_from_the_right() {
738 let ab = turned_span("ab", 0.0, 20.0, 6.0, 12.0, 10.0);
742 let cdef = span_with("cdef", 12.0, 20.0, 6.0, 12.0);
743 let words = words_from_blocks(&[block_of(vec![ab, cdef])]);
744 assert_eq!(words.len(), 1, "the split word did not join: {words:?}");
745 assert_eq!(words[0].text, "abcdef");
746 assert_eq!(words[0].dir, 0.0, "length lost to position");
747 }
748
749 #[test]
750 fn a_space_at_the_span_boundary_blocks_the_join() {
751 let hel = span_with("Hel ", 0.0, 20.0, 6.0, 12.0);
753 let lo = span_with("lo", 18.0, 20.0, 6.0, 12.0);
754 let words = words_from_blocks(&[block_of(vec![hel, lo])]);
755 assert_eq!(words.len(), 2, "the boundary space did not hold: {words:?}");
756 assert_eq!(words[0].text, "Hel");
757 assert_eq!(words[1].text, "lo");
758 }
759}