kobo_core/html_text/
extract.rs

1// SPDX-License-Identifier: MIT
2// Copyright (c) 2026 Nayeem Bin Ahsan
3//! DOM extraction: chapter XHTML -> plain text + per-block-element segments.
4
5use super::style::{self, IndentMap, MAX_INDENT_EM};
6use scraper::{Html, Selector};
7use std::sync::LazyLock;
8
9/// A run of chapter text belonging to one block element, with its char range.
10/// For `<img>`/`<figure>` segments the text range is empty (zero-width marker at
11/// the image's flow position) and `src`/`caption` carry the image data.
12#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
13pub struct TextSegment {
14    /// Inclusive start byte/char offset into the chapter text.
15    pub start: usize,
16    /// Exclusive end offset.
17    pub end: usize,
18    /// Element tag (e.g. `p`, `h1`, `li`, `img`, `figure`).
19    pub tag: String,
20    /// The element's `id` attribute if any (useful for anchoring).
21    pub id: Option<String>,
22    /// `src` for `img`/`figure` (None for text blocks).
23    pub src: Option<String>,
24    /// `figcaption` text for a `figure` (None for non-figures).
25    pub caption: Option<String>,
26    /// Left indent of the block, in `em`, resolved from the book's stylesheet
27    /// (see [`crate::html_text::style`]) plus any leading spaces the markup
28    /// kept. 0 for ordinary prose. Serde-defaulted so offset caches written
29    /// before indents existed still deserialize.
30    #[serde(default)]
31    pub indent: f32,
32    /// Bold/italic spans within this segment, in chapter-text offsets.
33    #[serde(default)]
34    pub styles: Vec<StyleRun>,
35}
36
37/// A run of emphasised text inside a chapter, in chapter-text byte offsets.
38///
39/// Held alongside the text rather than inside it. `Row` is a Slint struct
40/// carrying flat text, and threading styled spans through it would mean
41/// changing the `.slint` type, every construction site, and the justification
42/// and TTS-highlight code that keys off row ranges. Rows already carry byte
43/// ranges, so emphasis can simply be looked up by offset at draw time.
44#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
45pub struct StyleRun {
46    pub start: usize,
47    pub end: usize,
48    pub bold: bool,
49    pub italic: bool,
50}
51
52impl TextSegment {
53    /// Does this segment contain char offset `i`?
54    pub fn contains(&self, i: usize) -> bool {
55        i >= self.start && i < self.end
56    }
57}
58
59/// Selector for the block elements we extract as flow items (text + images).
60/// `image` matches SVG `<image>` (covers, inline SVG art).
61const BLOCK_SELECTOR: &str =
62    "p, h1, h2, h3, h4, h5, h6, li, blockquote, pre, div, figure, img, image";
63const CONTAINER_TAGS: &[&str] = &["div", "blockquote"];
64
65static BLOCK_SELECTOR_OBJ: LazyLock<Selector> =
66    LazyLock::new(|| Selector::parse(BLOCK_SELECTOR).expect("valid selector"));
67static IMG_SELECTOR_OBJ: LazyLock<Selector> =
68    LazyLock::new(|| Selector::parse("img").expect("selector"));
69static CAP_SELECTOR_OBJ: LazyLock<Selector> =
70    LazyLock::new(|| Selector::parse("figcaption").expect("selector"));
71
72/// Extract plain text + per-block-element segments from chapter XHTML.
73/// Images (`<img>`/`<figure>`) appear in document order as zero-width segments
74/// carrying `src`/`caption` (they contribute no text to the returned string).
75pub fn extract(xhtml: &str) -> (String, Vec<TextSegment>) {
76    extract_with_indents(xhtml, &IndentMap::new())
77}
78
79/// As [`extract`], but resolves each text block's left indent against the
80/// book's stylesheet (`class -> em`, built by [`style::parse_indents`]).
81///
82/// Calibre encodes code nesting as `margin-left` on a per-level class rather
83/// than as `<pre>`, so without the map every indent level collapses to zero and
84/// Python listings lose the structure that carries their meaning.
85pub fn extract_with_indents(xhtml: &str, indents: &IndentMap) -> (String, Vec<TextSegment>) {
86    let html = Html::parse_fragment(xhtml);
87    let sel = BLOCK_SELECTOR_OBJ.clone();
88    let child_sel = BLOCK_SELECTOR_OBJ.clone();
89    let img_sel = IMG_SELECTOR_OBJ.clone();
90    let cap_sel = CAP_SELECTOR_OBJ.clone();
91
92    let mut text = String::new();
93    let mut segs: Vec<TextSegment> = Vec::new();
94    let mut figure_srcs: std::collections::HashSet<String> = std::collections::HashSet::new();
95
96    for elem in html.select(&sel) {
97        let tag = elem.value().name().to_string();
98        let pos = text.len();
99        let id = elem.value().attr("id").map(|s| s.to_string());
100
101        let seg = match tag.as_str() {
102            "figure" => extract_figure_segment(elem, &img_sel, &cap_sel, pos, id, &mut figure_srcs),
103            "img" => {
104                if figure_srcs.contains(elem.value().attr("src").unwrap_or("")) {
105                    None
106                } else {
107                    Some(TextSegment {
108                        start: pos,
109                        end: pos,
110                        tag,
111                        id,
112                        src: elem.value().attr("src").map(|s| s.to_string()),
113                        caption: None,
114                        indent: 0.0,
115                        styles: Vec::new(),
116                    })
117                }
118            }
119            "image" => extract_svg_image_segment(elem, pos, id),
120            _ => extract_text_segment(elem, &child_sel, &mut text, pos, tag, id, &segs, indents),
121        };
122        if let Some(s) = seg {
123            segs.push(s);
124        }
125    }
126
127    let extra = scan_raw_svg_images(xhtml, &segs, &mut text);
128    segs.extend(extra);
129
130    (text, segs)
131}
132
133fn extract_figure_segment(
134    elem: scraper::ElementRef,
135    img_sel: &Selector,
136    cap_sel: &Selector,
137    pos: usize,
138    id: Option<String>,
139    figure_srcs: &mut std::collections::HashSet<String>,
140) -> Option<TextSegment> {
141    let src = elem
142        .select(img_sel)
143        .next()
144        .and_then(|i| i.value().attr("src"))
145        .map(|s| {
146            figure_srcs.insert(s.to_string());
147            s.to_string()
148        });
149    let cap = elem
150        .select(cap_sel)
151        .next()
152        .map(|c| c.text().collect::<String>().trim().to_string());
153    Some(TextSegment {
154        start: pos,
155        end: pos,
156        tag: "figure".to_string(),
157        id,
158        src,
159        caption: cap,
160        indent: 0.0,
161        styles: Vec::new(),
162    })
163}
164
165fn extract_svg_image_segment(
166    elem: scraper::ElementRef,
167    pos: usize,
168    id: Option<String>,
169) -> Option<TextSegment> {
170    let src = elem
171        .value()
172        .attr("xlink:href")
173        .or_else(|| elem.value().attr("href"))
174        .unwrap_or("");
175    if src.is_empty() {
176        return None;
177    }
178    Some(TextSegment {
179        start: pos,
180        end: pos,
181        tag: "image".to_string(),
182        id,
183        src: Some(src.to_string()),
184        caption: None,
185        indent: 0.0,
186        styles: Vec::new(),
187    })
188}
189
190fn extract_text_segment(
191    elem: scraper::ElementRef,
192    child_sel: &Selector,
193    text: &mut String,
194    pos: usize,
195    tag: String,
196    id: Option<String>,
197    segs: &[TextSegment],
198    indents: &IndentMap,
199) -> Option<TextSegment> {
200    if CONTAINER_TAGS.contains(&tag.as_str()) && elem.select(child_sel).next().is_some() {
201        return None;
202    }
203    let raw: String = elem.text().collect();
204    // `pre` keeps its indentation inside the text itself, so adding a block
205    // indent on top would double it.
206    let indent = if tag == "pre" {
207        0.0
208    } else {
209        block_indent_em(elem, &raw, indents)
210    };
211    // `pre` is verbatim: keep its own line breaks and leading indentation, only
212    // shedding the blank lines the markup wraps around the block. Every other
213    // block collapses to a trimmed run that the layout re-wraps.
214    let t: &str = if tag == "pre" {
215        raw.trim_matches('\n').trim_end()
216    } else {
217        raw.trim()
218    };
219    if t.is_empty() {
220        return None;
221    }
222    if !text.is_empty() {
223        text.push('\n');
224    }
225    let content_start = text.len();
226    text.push_str(t);
227    let end = text.len();
228    let dup = segs
229        .last()
230        .map(|s| text[s.start..s.end] == text[content_start..end])
231        .unwrap_or(false);
232    if dup {
233        text.truncate(pos);
234        return None;
235    }
236    // Emphasis offsets are measured in `raw`; `t` dropped the leading
237    // whitespace, so shift by however much went and clamp to the stored range.
238    let lead = raw.len() - raw.trim_start().len();
239    let styles = if tag == "pre" {
240        Vec::new()
241    } else {
242        collect_style_runs(elem)
243            .into_iter()
244            .filter_map(|(a, b, bold, italic)| {
245                let a = content_start + a.saturating_sub(lead);
246                let b = content_start + b.saturating_sub(lead);
247                let (a, b) = (a.max(content_start).min(end), b.max(content_start).min(end));
248                (a < b).then_some(StyleRun {
249                    start: a,
250                    end: b,
251                    bold,
252                    italic,
253                })
254            })
255            .collect()
256    };
257    Some(TextSegment {
258        start: content_start,
259        end,
260        tag,
261        id,
262        src: None,
263        caption: None,
264        indent,
265        styles,
266    })
267}
268
269fn is_bold_tag(name: &str) -> bool {
270    matches!(name, "b" | "strong")
271}
272
273fn is_italic_tag(name: &str) -> bool {
274    matches!(name, "i" | "em" | "cite" | "var")
275}
276
277/// Walk a block's descendants in document order, recording which byte ranges of
278/// its concatenated text sit inside a bold or italic element.
279///
280/// Offsets are into the raw (untrimmed) text of the block, which is the same
281/// string `elem.text()` produces -- both walk text nodes in the same order.
282fn collect_style_runs(elem: scraper::ElementRef) -> Vec<(usize, usize, bool, bool)> {
283    fn walk(
284        node: ego_tree::NodeRef<scraper::Node>,
285        bold: bool,
286        italic: bool,
287        pos: &mut usize,
288        out: &mut Vec<(usize, usize, bool, bool)>,
289    ) {
290        for child in node.children() {
291            match child.value() {
292                scraper::Node::Text(t) => {
293                    let start = *pos;
294                    *pos += t.len();
295                    if (bold || italic) && *pos > start {
296                        out.push((start, *pos, bold, italic));
297                    }
298                }
299                scraper::Node::Element(e) => {
300                    let name = e.name();
301                    walk(
302                        child,
303                        bold || is_bold_tag(name),
304                        italic || is_italic_tag(name),
305                        pos,
306                        out,
307                    );
308                }
309                _ => {}
310            }
311        }
312    }
313    let mut out = Vec::new();
314    let mut pos = 0usize;
315    walk(*elem, false, false, &mut pos, &mut out);
316    // Adjacent runs with the same styling are one run.
317    out.dedup_by(|b, a| {
318        if a.1 == b.0 && a.2 == b.2 && a.3 == b.3 {
319            a.1 = b.1;
320            true
321        } else {
322            false
323        }
324    });
325    out
326}
327
328/// Extra indent contributed by each leading space the markup preserved.
329///
330/// Calibre puts the coarse nesting level on the class (1em per level) but
331/// leaves finer, within-level alignment -- a continued call's arguments, a JSON
332/// key one step inside its brace -- as leading `&nbsp;`, which `trim` would
333/// otherwise discard. Half an em keeps that visible without competing with a
334/// real nesting step.
335const EM_PER_LEADING_SPACE: f32 = 0.5;
336
337/// Left indent for a text block: its class's stylesheet `margin-left` (or an
338/// inline one), plus whatever leading spaces the markup kept.
339///
340/// The two are additive because they encode different things -- see
341/// [`EM_PER_LEADING_SPACE`].
342fn block_indent_em(elem: scraper::ElementRef, raw: &str, indents: &IndentMap) -> f32 {
343    let from_class = elem
344        .value()
345        .attr("class")
346        .map(|classes| {
347            classes
348                .split_whitespace()
349                .filter_map(|c| indents.get(c).copied())
350                .fold(0.0f32, f32::max)
351        })
352        .unwrap_or(0.0);
353    let from_style = elem
354        .value()
355        .attr("style")
356        .and_then(style::inline_indent_em)
357        .unwrap_or(0.0);
358    let leading = raw
359        .chars()
360        .take_while(|c| *c == ' ' || *c == '\u{00A0}' || *c == '\t')
361        .count() as f32;
362    (from_class.max(from_style) + leading * EM_PER_LEADING_SPACE).min(MAX_INDENT_EM)
363}
364
365fn scan_raw_svg_images(xhtml: &str, segs: &[TextSegment], text: &mut String) -> Vec<TextSegment> {
366    let captured_srcs: std::collections::HashSet<String> =
367        segs.iter().filter_map(|s| s.src.clone()).collect();
368    let mut extra: Vec<TextSegment> = Vec::new();
369    let mut search = 0;
370    while let Some(pos) = xhtml[search..].find("<image") {
371        let abs = search + pos;
372        let tag_end = xhtml[abs..]
373            .find('>')
374            .map(|p| abs + p)
375            .unwrap_or(xhtml.len());
376        let tag = &xhtml[abs..tag_end];
377        for attr in &["xlink:href", "href"] {
378            if let Some(eq) = tag.find(attr) {
379                let rest = &tag[eq + attr.len()..];
380                let rest = rest.trim_start();
381                if let Some(after_eq) = rest.strip_prefix('=') {
382                    let v = after_eq.trim_start();
383                    let q = v.chars().next().unwrap_or('"');
384                    if (q == '"' || q == '\'') && v.len() > 1 {
385                        if let Some(end) = v[1..].find(q) {
386                            let src = &v[1..1 + end];
387                            if !captured_srcs.contains(src) && !src.is_empty() {
388                                extra.push(TextSegment {
389                                    start: text.len(),
390                                    end: text.len(),
391                                    tag: "image".to_string(),
392                                    id: None,
393                                    src: Some(src.to_string()),
394                                    caption: None,
395                                    indent: 0.0,
396                                    styles: Vec::new(),
397                                });
398                            }
399                            break;
400                        }
401                    }
402                }
403            }
404        }
405        search = tag_end;
406    }
407    extra
408}
409
410/// Find the [`TextSegment`] containing char offset `i`, if any.
411pub fn segment_at(segs: &[TextSegment], i: usize) -> Option<&TextSegment> {
412    segs.iter().find(|s| s.contains(i))
413}
414#[cfg(test)]
415mod tests;
kobo_core/html_text/extract.rs

kobo_core/html_text/
extract.rs
