Skip to main content

quarb_text/
lib.rs

1//! The text level: a shared, source-independent semantics for
2//! written documents — sections, paragraphs, quotes, lists, and
3//! verbatim blocks — produced by format crates and served by this
4//! crate's single adapter.
5//!
6//! The block model follows the atrep markup language (litogramma's
7//! koine core): every block is `(kind, taxis?, lemma?, body,
8//! hypograph?)` — the lemma is the head or title, the hypograph the
9//! footer or attribution, and a paragraph is the degenerate
10//! lemma-less, hypograph-less block. The format adapters (`quarb-text-html`,
11//! `quarb-text-markdown`, the built-in plain-text reader) lower
12//! their format into the [`Block`] event stream; this crate derives
13//! the section tree and implements the adapter once, so
14//! `//section[::lemma ...]`, `//paragraph`, and `//blockquote` read
15//! identically over any text substrate — including an atrep
16//! document mounted by `quarb-atrep`.
17//!
18//! - Node names are the structural kinds: `section`, `paragraph`,
19//!   `blockquote`, `unordered-list`, `ordered-list`,
20//!   `unordered-item`, `ordered-item`, `verbatim` — plus the
21//!   apparatus: `footnote` and `endnote` (callout and body share
22//!   the family name), `aside` (litogramma's fourth
23//!   deixis-attached family — anchored content, not apparatus),
24//!   `index-mark`, and the verse vocabulary
25//!  : `verse` holding `strophe`s holding `stichos`
26//!   lines, the stichos `::taxis` the citation coordinate.
27//! - `::lemma`, `::hypograph`, and `::taxis` are properties; bare
28//!   `::` (and `::text`) is the flattened prose of the subtree,
29//!   lemma first, hypograph last.
30//! - `::::level` on a section is the source heading level;
31//!   `::::lang` on a verbatim block is its declared language.
32//! - Sections are derived from the flat heading stream by the
33//!   outline rule: a heading closes every open section at its
34//!   level or deeper, then opens a section under the nearest
35//!   shallower one. Content before the first heading belongs to
36//!   the document root. A heading inside an open container
37//!   (blockquote, list) is decorative, not sectioning: it lowers
38//!   to a paragraph of its text.
39//! - Every kind admits `::lemma`, `::taxis`, and `::hypograph` —
40//!   the atrep model, where these are universal affordances of a
41//!   block rather than privileges of particular kinds.
42//! - Tables denormalize into nested lists: an `ordered-list`
43//!   carrying the `<table>` trait (`::lemma` = the caption), one
44//!   `ordered-item` per row (`::taxis` = row number, `<row>`
45//!   trait), one `unordered-item` per cell (`<cell>` trait) whose
46//!   `::lemma` is the column name — from the header row in grids,
47//!   from the row's `th` label otherwise; headerless cells carry
48//!   no lemma. A lemma'd item flattens as `lemma: prose`, so a
49//!   row exists, the bare cell text otherwise. Empty cells are
50//!   skipped.
51
52use quarb::{AstAdapter, NodeId, Value};
53
54mod ann;
55pub mod render;
56mod tier;
57pub use render::{Render, render_node, render_nodes};
58
59/// A block-level event in the text-level vocabulary — what a format
60/// adapter emits. Headings arrive flat; the section tree is
61/// derived here, once, for every adapter.
62#[derive(Debug, Clone, PartialEq)]
63pub enum Block {
64    /// A flat heading: `level` is the source level (`h2` → 2, a
65    /// LaTeX `\section` → its depth), `lemma` its text.
66    Heading { level: u8, lemma: String },
67    /// A plain paragraph — the implicit, lemma-less block.
68    Paragraph { text: String },
69    /// A speech: a paragraph whose lemma names the speaker (TEI's
70    /// `<said who>` / `<sp><speaker>`, litogramma's dialogue sim).
71    /// The reader sees a paragraph; `::lemma` answers the speaker
72    /// and the text stays the words spoken.
73    Dialogue {
74        lemma: String,
75        text: String,
76        /// The literary reading names the block `dialogue`
77        /// (litogramma's sim); the text level reads it as a
78        /// paragraph with a lemma.
79        lit: bool,
80    },
81    /// The literary reading (`lit:`): a prose block that is a
82    /// speech — litogramma's `dialogue-line` (at-drama `@:-`),
83    /// TEI's whole-paragraph `<said>` — with the unseen metadata
84    /// its aphanes monosims carry (`prosopon`, the speaker) as
85    /// fields and its genoses as traits.
86    Speech {
87        text: String,
88        genoses: Vec<String>,
89        fields: Vec<(String, String)>,
90    },
91    /// The literary reading: an inline span of the preceding flow
92    /// block — a quotation, an annotation (a name), an emphasis, a
93    /// stage direction — named by its litogramma sim, at byte
94    /// offsets into the block's normalized text, with its genoses
95    /// as traits and its aphanes monosims as fields. Spans nest by
96    /// containment of offsets: a name inside a speech is the
97    /// speech's child.
98    Span {
99        kind: String,
100        lo: u32,
101        hi: u32,
102        genoses: Vec<String>,
103        fields: Vec<(String, String)>,
104    },
105    /// An inline node of the preceding flow block with text of its
106    /// own that is NOT in the prose — the dictionary vocabulary's
107    /// `inflection`, a lookup key the print never displays: named
108    /// by `kind`, `<hidden>` beside its genoses, its fields as
109    /// properties, standing at the end of the block.
110    Hidden {
111        kind: String,
112        text: String,
113        genoses: Vec<String>,
114        fields: Vec<(String, String)>,
115    },
116    /// The literary reading: a milestone (`@("page:12")`) at an
117    /// offset into the preceding flow block — `::scheme`,
118    /// `::value`.
119    Milestone {
120        scheme: String,
121        value: String,
122        at: u32,
123    },
124    /// The literary reading: unseen metadata on the preceding
125    /// block itself (a paragraph's `eidos`).
126    Annotate { fields: Vec<(String, String)> },
127    /// The parsing pack: a token the source itself
128    /// parsed — litogramma's at-epimerismos diaphane, which TEI's
129    /// `w`/`pc`, the Russian National Corpus, OpenCorpora and
130    /// PROIEL import to — at byte offsets into the preceding flow
131    /// block, with its parsings (the first is the reading's; the
132    /// rest are alternatives) and the onym a head may point at.
133    Parsed {
134        lo: u32,
135        hi: u32,
136        onym: Option<String>,
137        parsings: Vec<Parsing>,
138    },
139    /// The parsing pack: a sentence the source declared (the
140    /// periodos diaphane, TEI's `s`), at byte offsets into the
141    /// preceding flow block, with the source's id.
142    Periodos { lo: u32, hi: u32, id: String },
143    /// The literary reading: a cast entry the source declares
144    /// (at-drama's `@:!! name … !!:@(id)`, a dramatis-persona
145    /// line, TEI's cast list) — the name, the id when the source
146    /// gives one, the description as the node's text.
147    Character {
148        lemma: String,
149        onym: Option<String>,
150        text: String,
151        genoses: Vec<String>,
152    },
153    /// The literary reading: genoses the preceding block wears as
154    /// traits — the sim of a paragraph-level simmere that has no
155    /// reading of its own (a block stage direction's paragraphs
156    /// read `paragraph<stage-direction>`).
157    Wear { genoses: Vec<String> },
158    /// Inline content belonging directly to the open container (a
159    /// list item's own text, a bare-text blockquote). With no open
160    /// container it is read as a paragraph.
161    Text { text: String },
162    /// Open a nesting container. Items take their `unordered-` /
163    /// `ordered-` flavor (and taxis) from the enclosing list.
164    Open {
165        kind: Container,
166        lemma: Option<String>,
167    },
168    /// Close the innermost open container, optionally with its
169    /// hypograph (footer or attribution).
170    Close { hypograph: Option<String> },
171    /// A verbatim block — code or other preformatted lines, kept
172    /// as authored.
173    Verbatim { lang: Option<String>, text: String },
174    /// An in-text note callout (the deixis): `onym`
175    /// names the body it cites within its family. Emitted after
176    /// the flow block it sits in; it becomes that block's child,
177    /// in order, and the block's own text stays clean of markers.
178    /// `family: None` means the source declares none (an HTML
179    /// noteref): the callout takes its resolved body's family,
180    /// and a dangling one defaults to footnote.
181    NoteRef {
182        onym: String,
183        family: Option<NoteFamily>,
184        /// The declared spelling placed this pair in the margin
185        /// (a Tufte-style sidenote): surfaced as `::::form =
186        /// "margin"` on both ends — the family stays footnote,
187        /// placement is presentation.
188        margin: bool,
189        /// The callout's byte offset in the flow block's text,
190        /// when the producer knows it (the literary reading's
191        /// inline notes): the citation in force at the note.
192        at: Option<u32>,
193    },
194    /// An index mark: an invisible anchor declaring
195    /// this place concerns `term`. The back-of-book index is a
196    /// query over these, never a stored structure.
197    IndexMark { term: String },
198    /// A reference mark — the text-level mention (html's in-prose
199    /// `<a href>`, markdown's `[text](url)`, LaTeX's `\ref`):
200    /// `target` is the identifier as written, `text` the authored
201    /// link text (LaTeX `\ref` has none), and `internal` is the
202    /// producer's declaration that the target names a label in
203    /// *this* document (`#fragment`, a `\ref` key) rather than
204    /// another document. A mention, not an attachment: refs do
205    /// not carry `<deixis>`.
206    Ref {
207        target: String,
208        text: Option<String>,
209        internal: bool,
210    },
211    /// A point anchor — an invisible in-flow node bearing an
212    /// `onym` at this position (LaTeX's mid-flow `\label`, an
213    /// inline html `id=`). The IndexMark construction: child of
214    /// its flow block, document order.
215    Anchor { onym: String },
216    /// A citation mark — the bibliography's mention (LaTeX's
217    /// `\cite`, koine's cite monosim): `target` is the authored
218    /// key, resolved against the `bib` bearers — a namespace of
219    /// its own, separate from labels (LaTeX precedent). The
220    /// `locator` is the mark's own pinpoint (`\cite[I.1]{key}`,
221    /// `[@key, I.1]`), kept as written.
222    Cite {
223        target: String,
224        locator: Option<String>,
225    },
226    /// One bibliographic entry: a block bearing its authored
227    /// `key`. Unstructured sources (\bibitem) carry the full
228    /// reference as `text`; structured ones (bibliogramma,
229    /// BibTeX/BibLaTeX through it) carry `fields` — canonical
230    /// campus name → value, Latin per the bibliogramma
231    /// vocabulary — and the entry's `genus` (liber,
232    /// commentarius, …).
233    Bib {
234        key: String,
235        text: String,
236        fields: Vec<(String, String)>,
237        genus: Option<String>,
238    },
239    /// A block label: the innermost open section takes `onym` as
240    /// the name it bears (a heading's `id`; a `\label` attached
241    /// to a sectioning command — the promotion rule). With no
242    /// open section it degrades to a point [`Block::Anchor`].
243    Label { onym: String },
244    /// A verse block (litogramma's stichoi model):
245    /// strophes of lines, denormalized like [`Block::Table`].
246    /// Lines arrive flattened by the adapter and are kept as
247    /// authored; the stichos `::taxis` numbers lines 1-based,
248    /// CONTINUOUSLY across strophes — the citation coordinate.
249    Verse {
250        lemma: Option<String>,
251        strophes: Vec<Vec<String>>,
252        hypograph: Option<String>,
253    },
254    /// A table, denormalized here into nested lists (rows =
255    /// ordered items with the `<row>` trait, cells = unordered
256    /// items with the `<cell>` trait and the column name as
257    /// `::lemma`). Header *detection* is the adapter's job; the
258    /// lowering rule lives here. A cell's own `label` (a row's
259    /// `th`) wins over the positional `headers` entry.
260    Table {
261        lemma: Option<String>,
262        headers: Option<Vec<String>>,
263        rows: Vec<Vec<Cell>>,
264    },
265}
266
267/// One table cell as an adapter hands it over: the text, plus the
268/// label a row-shaped dialect attaches directly (an infobox row's
269/// `th`). Grid dialects leave `label` empty and let the lowering
270/// zip the header row on by position.
271#[derive(Debug, Clone, PartialEq, Eq)]
272pub struct Cell {
273    pub label: Option<String>,
274    pub text: String,
275}
276
277impl From<&str> for Cell {
278    fn from(text: &str) -> Self {
279        Cell {
280            label: None,
281            text: text.to_string(),
282        }
283    }
284}
285
286impl From<String> for Cell {
287    fn from(text: String) -> Self {
288        Cell { label: None, text }
289    }
290}
291
292/// The nesting containers an adapter opens and closes explicitly.
293#[derive(Debug, Clone, PartialEq, Eq)]
294pub enum Container {
295    Blockquote,
296    UnorderedList,
297    /// `start` is the first item's ordinal (Markdown's `3.` lists).
298    OrderedList {
299        start: i64,
300    },
301    /// A list item; flavor and taxis come from the enclosing list.
302    Item,
303    /// A note body (the noted): opens a note node of
304    /// its family at the document root — litogramma's canonical
305    /// document-end placement — holding its own blocks. `margin`
306    /// as on [`Block::NoteRef`].
307    Note {
308        onym: String,
309        family: NoteFamily,
310        margin: bool,
311    },
312    /// The dictionary vocabulary: an entry, the open's lemma its
313    /// headword (which registers as its onym unless one is
314    /// given), the taxis its homograph number.
315    Entry {
316        taxis: Option<i64>,
317        onym: Option<String>,
318    },
319    /// A sense under an entry: the taxis its number at its depth,
320    /// the open's lemma a gloss-title when the source gives one.
321    Sense {
322        taxis: Option<i64>,
323    },
324}
325
326/// The deixis-attached families litogramma's canon names
327///: each family's callout and body
328/// share its name. `Aside` rides the same construction —
329/// koine's `aside` sim, an anchored body whose insertion point
330/// is a deixis — but is content, not apparatus: its bodies do
331/// not carry `<note>`.
332#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
333pub enum NoteFamily {
334    Footnote,
335    Endnote,
336    Aside,
337}
338
339impl NoteFamily {
340    fn kind(self) -> Kind {
341        match self {
342            NoteFamily::Footnote => Kind::Footnote,
343            NoteFamily::Endnote => Kind::Endnote,
344            NoteFamily::Aside => Kind::Aside,
345        }
346    }
347}
348
349/// One parsing of a source-parsed token, in the
350/// pack's own vocabulary: the lexeme's lemma, the part of speech
351/// (`UPOS` or `UPOS/XPOS`), the accidents (features, in the
352/// source's spelling), a semantic class, the head's onym and the
353/// dependency relation.
354#[derive(Debug, Clone, Default, PartialEq, Eq)]
355pub struct Parsing {
356    pub lexema: Option<String>,
357    pub meros: Option<String>,
358    pub parepomena: Option<String>,
359    pub semasia: Option<String>,
360    pub kephale: Option<String>,
361    pub schesis: Option<String>,
362}
363
364/// A source-parsed token waiting for the tokenizer: its byte
365/// range in the block's prose, its onym, its parsings.
366#[derive(Debug, Clone)]
367struct DeclaredToken {
368    lo: u32,
369    hi: u32,
370    onym: Option<String>,
371    parsings: Vec<Parsing>,
372}
373
374/// A row of a cast table: the character's id (the
375/// `prosopon` a mention answers), its name, the pattern a word
376/// token must match to be its mention, and the table's other
377/// columns as the character's fields.
378#[derive(Debug, Clone)]
379pub struct CastRow {
380    pub id: String,
381    pub name: String,
382    pub pattern: Option<regex::Regex>,
383    /// How many word tokens the pattern spans (a `mention` or
384    /// `stem` of several words matches a run of tokens).
385    pub words: usize,
386    pub fields: Vec<(String, String)>,
387}
388
389impl CastRow {
390    /// Read a cast table from a CSV file: an `id` column (required),
391    /// a `name`, and the pattern a word token must match to be the
392    /// character's mention — `pattern` as a regex, `stem` (or
393    /// `prefix`) as a prefix, or `mention` / `form` as the whole
394    /// word — with every
395    /// other column kept as the character's fields.
396    pub fn read_csv(path: &str) -> Result<Vec<CastRow>, String> {
397        let mut reader = csv::Reader::from_path(path).map_err(|e| e.to_string())?;
398        let headers: Vec<String> = reader
399            .headers()
400            .map_err(|e| e.to_string())?
401            .iter()
402            .map(str::to_string)
403            .collect();
404        if !headers.iter().any(|h| h == "id") {
405            return Err(format!(
406                "a cast table needs an id column (found: {})",
407                headers.join(", ")
408            ));
409        }
410        let mut rows = Vec::new();
411        for record in reader.records() {
412            let record = record.map_err(|e| e.to_string())?;
413            let get = |k: &str| {
414                headers
415                    .iter()
416                    .position(|h| h == k)
417                    .and_then(|i| record.get(i))
418                    .map(str::trim)
419                    .filter(|v| !v.is_empty())
420            };
421            let Some(id) = get("id") else { continue };
422            let (pattern, words) = if let Some(p) = get("pattern") {
423                let re = regex::Regex::new(p).map_err(|e| format!("cast pattern for {id}: {e}"))?;
424                // A pattern of several words matches a run of that
425                // many tokens, as a stem or a mention does.
426                (Some(re), p.split_whitespace().count().max(1))
427            } else if let Some(stem) = get("stem").or_else(|| get("prefix")) {
428                let re = regex::Regex::new(&format!("^{}", regex::escape(stem))).unwrap();
429                (Some(re), stem.split_whitespace().count())
430            } else if let Some(form) = get("mention").or_else(|| get("form")) {
431                let re = regex::Regex::new(&format!("^{}$", regex::escape(form))).unwrap();
432                (Some(re), form.split_whitespace().count())
433            } else {
434                (None, 0)
435            };
436            let fields: Vec<(String, String)> = headers
437                .iter()
438                .zip(record.iter())
439                .filter(|(h, v)| h.as_str() != "id" && !v.trim().is_empty())
440                .map(|(h, v)| (h.clone(), v.trim().to_string()))
441                .collect();
442            rows.push(CastRow {
443                id: id.to_string(),
444                name: get("name").unwrap_or(id).to_string(),
445                pattern,
446                words,
447                fields,
448            });
449        }
450        Ok(rows)
451    }
452}
453
454/// The structural kind of a node — also its name.
455#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
456enum Kind {
457    Document,
458    Section,
459    /// The literary reading's cast entry: a character
460    /// declared by the source or by a cast table, `::lemma` its
461    /// name, `::onym` its id, `<-character` its mentions.
462    Character,
463    /// The literary reading's gazetteer entry: a place
464    /// declared by a places table, `::lemma` its name, `::onym` its
465    /// id, `<-place` its mentions.
466    Place,
467    Paragraph,
468    Blockquote,
469    UnorderedList,
470    OrderedList,
471    UnorderedItem,
472    OrderedItem,
473    Verbatim,
474    /// Callout and body both — they share the name by
475    /// construction (litogramma F5): `//footnote` gathers the
476    /// whole apparatus, `<deixis>`/`<note>` tell them apart.
477    Footnote,
478    /// The second note family, same construction.
479    Endnote,
480    /// The anchored-content family, same construction
481    /// (litogramma's aside): body + insertion-point deixis. Content, not
482    /// apparatus — no `<note>` on its bodies.
483    Aside,
484    /// An index mark: `::term`, in flow position.
485    IndexMark,
486    /// A reference mark: `::target`, `-->` resolves it.
487    Ref,
488    /// A citation mark: `::target` the authored key, resolved
489    /// against the `bib` namespace.
490    Cit,
491    /// One bibliographic entry: `::onym` the key it bears, its
492    /// projection the full reference text.
493    Bib,
494    /// A point anchor: `::onym`, an invisible in-flow bearer.
495    Anchor,
496    /// The verse vocabulary: the stichoi container…
497    Verse,
498    /// …its strophes…
499    Strophe,
500    /// …and its lines, `::taxis` the citation coordinate.
501    Stichos,
502    /// The corpus reading: one token of a prose
503    /// block — a UAX #29 word segment — with `::` its form,
504    /// `::lower` the case-folded form, `<word>` / `<number>` /
505    /// `<punct>` its class, `::::n` its position in the document
506    /// and `::::sentence` the UAX #29 sentence it falls in. The
507    /// tokens sit under the block's derived `sentence` nodes
508    /// — the sentence tier — and a treebank's
509    /// A treebank's sentence: in a standalone CoNLL-U
510    /// document the file's blank-line blocks are declared
511    /// structure that nothing re-segments, so each is a prose
512    /// block — `::` its text (`# text`, else the forms joined per
513    /// `SpaceAfter`), `::id` its `# sent_id`, every other `# key =
514    /// value` comment a property — and under `corpus:` its tokens
515    /// are its children. Over prose, a sidecar's sentence stays an
516    /// annotation on the token.
517    Sentence,
518    /// The literary reading's speech block: litogramma's
519    /// `dialogue-line`.
520    Speech,
521    /// The literary reading's drama speech (`@: Speaker`): the
522    /// lemma is the printed speech prefix.
523    Dialogue,
524    /// The literary reading's inline span, named by its sim
525    /// (`quotation`, `annotation`, `emphasis`, …).
526    Span,
527    /// The literary reading's milestone, a point in the prose.
528    Milestone,
529    /// The dictionary vocabulary: a record named by its headword.
530    Entry,
531    /// A sense under an entry, nested as the dictionary subdivides.
532    Sense,
533}
534
535impl Kind {
536    fn name(self) -> Option<&'static str> {
537        Some(match self {
538            Kind::Document => return None,
539            Kind::Section => "section",
540            Kind::Paragraph => "paragraph",
541            Kind::Blockquote => "blockquote",
542            Kind::Footnote => "footnote",
543            Kind::Endnote => "endnote",
544            Kind::Aside => "aside",
545            Kind::IndexMark => "index-mark",
546            Kind::Ref => "ref",
547            Kind::Cit => "cit",
548            Kind::Bib => "bib",
549            Kind::Anchor => "anchor",
550            Kind::Verse => "verse",
551            Kind::Strophe => "strophe",
552            Kind::Stichos => "stichos",
553            Kind::UnorderedList => "unordered-list",
554            Kind::OrderedList => "ordered-list",
555            Kind::UnorderedItem => "unordered-item",
556            Kind::OrderedItem => "ordered-item",
557            Kind::Verbatim => "verbatim",
558            Kind::Sentence => "sentence",
559            Kind::Speech => "dialogue-line",
560            Kind::Character => "character",
561            Kind::Place => "place",
562            Kind::Dialogue => "dialogue",
563            // The adapter answers the sim name kept on the node.
564            Kind::Span => "span",
565            Kind::Milestone => "milestone",
566            Kind::Entry => "entry",
567            Kind::Sense => "sense",
568        })
569    }
570}
571
572struct Node {
573    kind: Kind,
574    lemma: Option<String>,
575    hypograph: Option<String>,
576    taxis: Option<i64>,
577    /// Source heading level, on sections.
578    level: Option<u8>,
579    /// Declared language, on verbatim blocks.
580    lang: Option<String>,
581    /// First ordinal of an ordered list (not exposed; feeds the
582    /// items' taxis).
583    start: i64,
584    /// The node's own (direct) text, before subtree flattening.
585    text: String,
586    /// The flattened prose of the subtree — the `::` projection.
587    prose: String,
588    /// The node heads a denormalized table (`<table>` trait).
589    table: bool,
590    /// The node is a denormalized table row (`<row>` trait).
591    row: bool,
592    /// The node is a denormalized table cell (`<cell>` trait).
593    cell: bool,
594    /// The apparatus: the note name as written —
595    /// and, on an index mark, the term.
596    onym: Option<String>,
597    /// A footnote callout (`<deixis>`) rather than a body.
598    deixis: bool,
599    /// A callout whose body is missing (`<dangling>`).
600    dangling: bool,
601    /// A callout whose source declared no family: resolution may
602    /// re-kind it from the body it reaches.
603    family_open: bool,
604    /// The declared spelling was a margin form (`::::form`).
605    margin: bool,
606    /// Callout -> body edge, once resolved.
607    note_edge: Option<NodeId>,
608    /// Body <- callouts (the reverse index).
609    cites: Vec<NodeId>,
610    /// A ref's target identifier, as written.
611    target: Option<String>,
612    /// A cit's locator (page, chapter, line), as written.
613    locator: Option<String>,
614    /// The producer declared the target in-document.
615    internal: bool,
616    /// A structured bib entry's fields (campus → value) and its
617    /// genus, per the bibliogramma vocabulary.
618    fields: Vec<(String, String)>,
619    genus: Option<String>,
620    /// Ref -> bearer edge, once resolved (in-document).
621    ref_edge: Option<NodeId>,
622    /// Bearer <- refs (the reverse index).
623    ref_cites: Vec<NodeId>,
624    parent: Option<NodeId>,
625    children: Vec<NodeId>,
626    /// The corpus reading's annotation, on token nodes.
627    /// A sentence the corpus reading derived at mount (ruling
628    /// #64) — from the segmentation in force, never declared by
629    /// the source — as against a treebank's declared one.
630    derived: bool,
631    /// The literary reading: the genoses a simmere wore (its
632    /// traits), the sim name of a span, its byte range in the
633    /// flow block's text, and the tokens it covers.
634    genoses: Vec<String>,
635    span_kind: Option<String>,
636    at: Option<(u32, u32)>,
637    span_tokens: Vec<NodeId>,
638    /// The character a mention resolves to (`->character`), and on
639    /// a character the mentions that resolve to it (`<-character`).
640    character: Option<NodeId>,
641    mentions: Vec<NodeId>,
642    /// The place a mention resolves to (`->place`); a place keeps
643    /// its mentions in `mentions` too.
644    place: Option<NodeId>,
645    /// The parsing pack's tokens and sentences declared on this
646    /// flow block, consumed by the tokenizer.
647    declared_tokens: Vec<DeclaredToken>,
648    declared_sentences: Vec<(u32, u32, String)>,
649}
650
651/// A token's class — the trait it wears.
652#[derive(Debug, Clone, Copy, PartialEq, Eq)]
653enum TokenClass {
654    /// Carries a letter.
655    Word,
656    /// Carries a digit and no letter.
657    Number,
658    /// Everything else that is not whitespace.
659    Punct,
660}
661
662impl TokenClass {
663    fn name(self) -> &'static str {
664        match self {
665            TokenClass::Word => "word",
666            TokenClass::Number => "number",
667            TokenClass::Punct => "punct",
668        }
669    }
670}
671
672/// A mention: a span of tokens an annotation marks as
673/// a named entity or a coreference mention — the sentence pattern
674/// again, an annotation on the tokens, never a container. Decoded
675/// at mount from a `ner` key in MISC (BIO and BIOES tags) and from
676/// CorefUD's `Entity=` bracket notation.
677#[derive(Debug, Clone)]
678struct Mention {
679    tokens: Vec<NodeId>,
680    /// The entity type (PERSON, GPE, …), when the source names one.
681    etype: Option<String>,
682    /// The coreference cluster id, when the source declares one.
683    cluster: Option<String>,
684}
685
686/// The bracket entries of a CorefUD `Entity=` value, which may
687/// run together (`(e4-object-5-(e3-person-3-`, `e3)(e5-x-1)`) or
688/// be `|`-separated: an opening entry runs from its `(` to the
689/// next bracket, a closing one ends at its `)`.
690fn entity_entries(spec: &str) -> Vec<String> {
691    let mut out = Vec::new();
692    for piece in spec.split('|') {
693        let mut cur = String::new();
694        for ch in piece.chars() {
695            if ch == '(' && !cur.is_empty() {
696                out.push(std::mem::take(&mut cur));
697            }
698            cur.push(ch);
699            if ch == ')' {
700                out.push(std::mem::take(&mut cur));
701            }
702        }
703        if !cur.is_empty() {
704            out.push(cur);
705        }
706    }
707    out
708}
709
710/// `Key=Value|Key=Value` (FEATS, MISC) as pairs; an entry without
711/// `=` keeps its text as the key with an empty value; `_` is none.
712fn key_values(column: &str) -> Vec<(String, String)> {
713    if column == "_" || column.is_empty() {
714        return Vec::new();
715    }
716    column
717        .split('|')
718        .filter(|e| !e.is_empty())
719        .map(|e| match e.split_once('=') {
720            Some((k, v)) => (k.to_string(), v.to_string()),
721            None => (e.to_string(), String::new()),
722        })
723        .collect()
724}
725
726/// The UAX #29 sentence spans of a block's prose — with the
727/// sentence bonds applied when a `.desm` set is in force (the
728/// `syndesmos` crate: abbreviations a sentence may end in without
729/// ending, and patterns whose match straddles a break).
730fn sentence_spans(prose: &str, bonds: Option<&syndesmos::Syndesmos>) -> Vec<(usize, usize)> {
731    let raw: Vec<(usize, &str)> = match bonds {
732        Some(b) => b.split_sentence_bound_indices(prose),
733        None => {
734            use unicode_segmentation::UnicodeSegmentation;
735            prose.split_sentence_bound_indices().collect()
736        }
737    };
738    raw.into_iter()
739        .filter(|(_, seg)| !seg.trim().is_empty())
740        .map(|(at, seg)| (at, at + seg.len()))
741        .collect()
742}
743
744/// The ten standard CoNLL-U columns, in order.
745const CONLLU_COLUMNS: [&str; 10] = [
746    "ID", "FORM", "LEMMA", "UPOS", "XPOS", "FEATS", "HEAD", "DEPREL", "DEPS", "MISC",
747];
748
749/// One token line of a CoNLL-U sentence, as read.
750#[derive(Debug, Clone, Default)]
751struct ConlluToken {
752    id: String,
753    /// A multiword range `a-b`, parsed.
754    range: Option<(u32, u32)>,
755    /// The numeric id of a word, or an empty node's integer part.
756    ord: u32,
757    /// An empty node (`8.1`).
758    empty: bool,
759    form: String,
760    lemma: String,
761    upos: String,
762    xpos: String,
763    feats: String,
764    head: String,
765    deprel: String,
766    deps: String,
767    misc: String,
768    /// CoNLL-U Plus columns beyond the ten, under their declared
769    /// names with `:` read as `-`.
770    extra: Vec<(String, String)>,
771}
772
773/// One sentence block of a CoNLL-U file.
774#[derive(Debug, Clone, Default)]
775struct ConlluSentence {
776    /// `# key = value` comments, in order (`sent_id`, `text`, …).
777    comments: Vec<(String, String)>,
778    /// `# newdoc [id = X]` opened a document before this sentence.
779    newdoc: Option<Option<String>>,
780    /// `# newpar [id = X]` opened a paragraph before this sentence.
781    newpar: Option<Option<String>>,
782    tokens: Vec<ConlluToken>,
783}
784
785/// A CoNLL-U file: its sentences and its file-level declarations
786/// (`# global.columns`, `# global.Entity`).
787#[derive(Debug, Clone, Default)]
788struct ConlluDoc {
789    sentences: Vec<ConlluSentence>,
790    globals: Vec<(String, String)>,
791}
792
793/// Whether `text` reads as CoNLL-U: its first line that is neither
794/// blank nor a `#` comment is a tab-separated token line whose id
795/// is a word (`3`), a range (`3-4`) or an empty node (`3.1`), with
796/// the ten columns (or, under a `# global.columns` header, that
797/// header's count). The sniff for an extensionless input or a
798/// pipe, where no `.conllu` names the reading.
799pub fn looks_like_conllu(text: &str) -> bool {
800    let mut columns = CONLLU_COLUMNS.len();
801    for line in text.lines().take(200) {
802        let line = line.trim_end_matches('\r');
803        if line.trim().is_empty() {
804            continue;
805        }
806        if let Some(c) = line.strip_prefix('#') {
807            if let Some((k, v)) = c.split_once('=')
808                && k.trim() == "global.columns"
809            {
810                columns = v.split_whitespace().count().max(2);
811            }
812            continue;
813        }
814        let fields: Vec<&str> = line.split('\t').collect();
815        if fields.len() != columns {
816            return false;
817        }
818        let id = fields[0];
819        let (a, b) = match id.split_once(['-', '.']) {
820            Some((a, b)) => (a, Some(b)),
821            None => (id, None),
822        };
823        return a.parse::<u32>().is_ok() && b.is_none_or(|b| b.parse::<u32>().is_ok());
824    }
825    false
826}
827
828/// The treebank files of a directory, for the set readings: every
829/// `.conllu` / `.conllup` beneath `dir` (a UD treebank's train /
830/// dev / test, a release's languages), named by their path
831/// relative to `dir`, in path order. Hidden entries are skipped.
832/// An empty set is an error: the directory holds no treebank.
833pub fn read_conllu_dir(dir: &std::path::Path) -> Result<Vec<(String, String)>, String> {
834    read_conllu_dir_with(dir, &quarb::source::SourceDecoding::default())
835}
836
837/// [`read_conllu_dir`] under a mount's source-boundary rule: each
838/// file decodes by `?encoding=`, a byte-order mark, else strict
839/// UTF-8 — refused naming the file otherwise.
840pub fn read_conllu_dir_with(
841    dir: &std::path::Path,
842    decoding: &quarb::source::SourceDecoding,
843) -> Result<Vec<(String, String)>, String> {
844    fn walk(
845        dir: &std::path::Path,
846        base: &std::path::Path,
847        decoding: &quarb::source::SourceDecoding,
848        out: &mut Vec<(String, String)>,
849    ) -> Result<(), String> {
850        let mut entries: Vec<std::path::PathBuf> = std::fs::read_dir(dir)
851            .map_err(|e| format!("reading {}: {e}", dir.display()))?
852            .filter_map(|e| e.ok().map(|e| e.path()))
853            .filter(|p| {
854                !p.file_name()
855                    .and_then(|n| n.to_str())
856                    .is_some_and(|n| n.starts_with('.'))
857            })
858            .collect();
859        entries.sort();
860        for p in entries {
861            if p.is_dir() {
862                walk(&p, base, decoding, out)?;
863            } else if p.extension().and_then(|e| e.to_str()).is_some_and(|e| {
864                e.eq_ignore_ascii_case("conllu") || e.eq_ignore_ascii_case("conllup")
865            }) {
866                let bytes =
867                    std::fs::read(&p).map_err(|e| format!("reading {}: {e}", p.display()))?;
868                let text = decoding
869                    .decode(&p.display().to_string(), &bytes)
870                    .map_err(|e| format!("reading {}: {e}", p.display()))?
871                    .text;
872                let text = text
873                    .strip_prefix('\u{feff}')
874                    .map(str::to_owned)
875                    .unwrap_or(text);
876                let name = p
877                    .strip_prefix(base)
878                    .unwrap_or(&p)
879                    .to_string_lossy()
880                    .into_owned();
881                out.push((name, text));
882            }
883        }
884        Ok(())
885    }
886    let mut out = Vec::new();
887    walk(dir, dir, decoding, &mut out)?;
888    if out.is_empty() {
889        return Err(format!("{} holds no .conllu file", dir.display()));
890    }
891    Ok(out)
892}
893
894/// The sentences of a CoNLL-U text: token lines split on tabs,
895/// sentences separated by blank lines, comments read (`# key =
896/// value` kept on the sentence, `# newdoc` / `# newpar` as
897/// structure, `# global.*` on the file), empty nodes (`1.1`) kept,
898/// multiword ranges (`1-2`) kept as the unit that aligns to the
899/// text. A `# global.columns` line (CoNLL-U Plus) reorders the
900/// columns and names extra ones.
901fn parse_conllu(text: &str) -> Result<ConlluDoc, String> {
902    let mut doc = ConlluDoc::default();
903    let mut current = ConlluSentence::default();
904    let mut columns: Vec<String> = CONLLU_COLUMNS.iter().map(|c| c.to_string()).collect();
905    let trimmed = |s: &str| s.trim().to_string();
906    for (i, line) in text.lines().enumerate() {
907        let line = line.trim_end_matches('\r');
908        if line.trim().is_empty() {
909            if !current.tokens.is_empty() {
910                doc.sentences.push(std::mem::take(&mut current));
911            }
912            continue;
913        }
914        if let Some(comment) = line.strip_prefix('#') {
915            let comment = comment.trim();
916            if let Some(rest) = comment.strip_prefix("newdoc") {
917                let id = rest
918                    .trim()
919                    .strip_prefix("id")
920                    .and_then(|r| r.trim().strip_prefix('='));
921                current.newdoc = Some(id.map(trimmed));
922            } else if let Some(rest) = comment.strip_prefix("newpar") {
923                let id = rest
924                    .trim()
925                    .strip_prefix("id")
926                    .and_then(|r| r.trim().strip_prefix('='));
927                current.newpar = Some(id.map(trimmed));
928            } else if let Some((k, v)) = comment.split_once('=') {
929                let (k, v) = (k.trim(), v.trim());
930                if let Some(g) = k.strip_prefix("global.") {
931                    if g == "columns" {
932                        columns = v.split_whitespace().map(|c| c.to_string()).collect();
933                    }
934                    doc.globals.push((g.to_string(), v.to_string()));
935                } else {
936                    current.comments.push((k.to_string(), v.to_string()));
937                }
938            }
939            continue;
940        }
941        let f: Vec<&str> = line.split('\t').collect();
942        if f.len() < 2 {
943            return Err(format!(
944                "CoNLL-U line {}: expected tab-separated columns",
945                i + 1
946            ));
947        }
948        let mut tok = ConlluToken::default();
949        for (k, name) in columns.iter().enumerate() {
950            let v = f.get(k).copied().unwrap_or("_");
951            match name.to_ascii_uppercase().as_str() {
952                "ID" => tok.id = v.to_string(),
953                "FORM" => tok.form = v.to_string(),
954                "LEMMA" => tok.lemma = v.to_string(),
955                "UPOS" => tok.upos = v.to_string(),
956                "XPOS" => tok.xpos = v.to_string(),
957                "FEATS" => tok.feats = v.to_string(),
958                "HEAD" => tok.head = v.to_string(),
959                "DEPREL" => tok.deprel = v.to_string(),
960                "DEPS" => tok.deps = v.to_string(),
961                "MISC" => tok.misc = v.to_string(),
962                _ => {
963                    if v != "_" && !v.is_empty() {
964                        tok.extra.push((name.replace(':', "-"), v.to_string()));
965                    }
966                }
967            }
968        }
969        if tok.id.is_empty() {
970            return Err(format!("CoNLL-U line {}: no token id", i + 1));
971        }
972        if let Some((a, b)) = tok.id.split_once('-') {
973            match (a.parse::<u32>(), b.parse::<u32>()) {
974                (Ok(a), Ok(b)) if a <= b => {
975                    tok.range = Some((a, b));
976                    tok.ord = a;
977                }
978                _ => {
979                    return Err(format!(
980                        "CoNLL-U line {}: malformed multiword range {:?}",
981                        i + 1,
982                        tok.id
983                    ));
984                }
985            }
986        } else if let Some((a, _)) = tok.id.split_once('.') {
987            tok.empty = true;
988            tok.ord = a.parse().map_err(|_| {
989                format!(
990                    "CoNLL-U line {}: malformed empty-node id {:?}",
991                    i + 1,
992                    tok.id
993                )
994            })?;
995        } else {
996            tok.ord = tok
997                .id
998                .parse()
999                .map_err(|_| format!("CoNLL-U line {}: malformed token id {:?}", i + 1, tok.id))?;
1000        }
1001        current.tokens.push(tok);
1002    }
1003    if !current.tokens.is_empty() {
1004        doc.sentences.push(current);
1005    }
1006    Ok(doc)
1007}
1008
1009/// The sentence's surface text from its tokens: forms in order, a
1010/// multiword range standing for its words, a space after each
1011/// unless `SpaceAfter=No`; empty nodes are not in the text.
1012fn conllu_surface(tokens: &[ConlluToken]) -> String {
1013    let mut out = String::new();
1014    let mut covered: u32 = 0; // the last word id a range covers
1015    for t in tokens {
1016        if t.empty || (t.range.is_none() && t.ord <= covered) {
1017            continue;
1018        }
1019        if let Some((_, b)) = t.range {
1020            covered = b;
1021        }
1022        out.push_str(&t.form);
1023        let no_space = key_values(&t.misc)
1024            .iter()
1025            .any(|(k, v)| k.eq_ignore_ascii_case("SpaceAfter") && v == "No");
1026        if !no_space {
1027            out.push(' ');
1028        }
1029    }
1030    out.trim_end().to_string()
1031}
1032
1033fn class_of(seg: &str) -> TokenClass {
1034    if seg.chars().any(char::is_alphabetic) {
1035        TokenClass::Word
1036    } else if seg.chars().any(char::is_numeric) {
1037        TokenClass::Number
1038    } else {
1039        TokenClass::Punct
1040    }
1041}
1042
1043/// The class an annotation declares: UPOS `PUNCT` is
1044/// punctuation and `NUM` a number whatever the form's letters say
1045/// (`3rd`, a tagger's `PUNCT` on a lettered token); every other
1046/// tag, or none, leaves the form to decide.
1047fn class_of_annotated(form: &str, upos: &str) -> TokenClass {
1048    match upos {
1049        "PUNCT" => TokenClass::Punct,
1050        "NUM" => TokenClass::Number,
1051        _ => class_of(form),
1052    }
1053}
1054
1055/// Whether a token's form is a quotation mark: every
1056/// character one of Unicode's initial and final quotation
1057/// punctuation, the ASCII quotes, the guillemets, or the CJK
1058/// corner brackets. Such a token wears `<quote>` beside `<punct>`.
1059fn is_quote(form: &str) -> bool {
1060    !form.is_empty()
1061        && form.chars().all(|c| {
1062            matches!(
1063                c,
1064                '"' | '\'' | '`'
1065                    | '\u{AB}' | '\u{BB}'
1066                    | '\u{2018}'..='\u{201F}'
1067                    | '\u{2039}' | '\u{203A}'
1068                    | '\u{300C}'..='\u{300F}'
1069                    | '\u{301D}'..='\u{301F}'
1070                    | '\u{FF02}' | '\u{FF07}'
1071            )
1072        })
1073}
1074
1075impl Node {
1076    fn new(kind: Kind, parent: Option<NodeId>) -> Self {
1077        Node {
1078            kind,
1079            derived: false,
1080            lemma: None,
1081            hypograph: None,
1082            taxis: None,
1083            level: None,
1084            lang: None,
1085            start: 1,
1086            text: String::new(),
1087            prose: String::new(),
1088            table: false,
1089            row: false,
1090            cell: false,
1091            onym: None,
1092            deixis: false,
1093            dangling: false,
1094            family_open: false,
1095            margin: false,
1096            note_edge: None,
1097            cites: Vec::new(),
1098            target: None,
1099            locator: None,
1100            internal: false,
1101            fields: Vec::new(),
1102            genus: None,
1103            ref_edge: None,
1104            ref_cites: Vec::new(),
1105            parent,
1106            children: Vec::new(),
1107            genoses: Vec::new(),
1108            span_kind: None,
1109            at: None,
1110            span_tokens: Vec::new(),
1111            character: None,
1112            place: None,
1113            mentions: Vec::new(),
1114            declared_tokens: Vec::new(),
1115            declared_sentences: Vec::new(),
1116        }
1117    }
1118}
1119
1120/// Where an inline node met while flattening lands: the last flow
1121/// block, or — the dictionary vocabulary — the entry or sense
1122/// opened after it (the node was met in the headword line, whose
1123/// text is the lemma; the flag says so).
1124fn inline_host(
1125    nodes: &[Node],
1126    containers: &[NodeId],
1127    last_flow: Option<NodeId>,
1128) -> Option<(NodeId, bool)> {
1129    let container = containers
1130        .last()
1131        .copied()
1132        .filter(|c| matches!(nodes[c.0 as usize].kind, Kind::Entry | Kind::Sense));
1133    match (container, last_flow) {
1134        (Some(c), Some(f)) if c.0 > f.0 => Some((c, true)),
1135        (Some(c), None) => Some((c, true)),
1136        (_, Some(f)) => Some((f, false)),
1137        (None, None) => None,
1138    }
1139}
1140
1141/// The reference pass: collect every borne name (block labels
1142/// and point anchors — deixis callouts pair within their own
1143/// families and stay out of this namespace), then land each
1144/// internal ref on its bearer, `<dangling>` when nothing bears
1145/// the name. External refs resolve at query time, through the
1146/// engine's reference machinery.
1147fn resolve_refs(nodes: &mut [Node]) -> std::collections::HashMap<String, NodeId> {
1148    let mut onyms: std::collections::HashMap<String, NodeId> = std::collections::HashMap::new();
1149    // Citation keys are a namespace of their own, separate from
1150    // labels (LaTeX's \bibcite vs \newlabel precedent): `cit`
1151    // resolves against `bib` bearers only.
1152    let mut bibs: std::collections::HashMap<String, NodeId> = std::collections::HashMap::new();
1153    for (i, n) in nodes.iter().enumerate() {
1154        if n.deixis || matches!(n.kind, Kind::IndexMark | Kind::Ref | Kind::Cit) {
1155            continue;
1156        }
1157        if n.kind == Kind::Bib {
1158            if let Some(k) = &n.onym {
1159                bibs.entry(k.clone()).or_insert(NodeId(i as u64));
1160            }
1161            continue;
1162        }
1163        let bearer = n.kind == Kind::Anchor
1164            || (!matches!(n.kind, Kind::Footnote | Kind::Endnote | Kind::Aside)
1165                && n.onym.is_some());
1166        if bearer && let Some(o) = &n.onym {
1167            onyms.entry(o.clone()).or_insert(NodeId(i as u64));
1168        }
1169    }
1170    for i in 0..nodes.len() {
1171        let (map, key) = match nodes[i].kind {
1172            Kind::Ref if nodes[i].internal => (
1173                &onyms,
1174                nodes[i]
1175                    .target
1176                    .as_deref()
1177                    .map(|t| t.trim_start_matches('#').to_string())
1178                    .unwrap_or_default(),
1179            ),
1180            Kind::Cit => (&bibs, nodes[i].target.clone().unwrap_or_default()),
1181            _ => continue,
1182        };
1183        match map.get(&key) {
1184            Some(&bearer) => {
1185                nodes[i].ref_edge = Some(bearer);
1186                let me = NodeId(i as u64);
1187                nodes[bearer.0 as usize].ref_cites.push(me);
1188            }
1189            None => nodes[i].dangling = true,
1190        }
1191    }
1192    onyms
1193}
1194
1195/// Collapse whitespace runs to single spaces and trim — the prose
1196/// normalization adapters apply to inline content. Verbatim text
1197/// is the exception: it is kept as authored.
1198pub fn normalize_ws(s: &str) -> String {
1199    s.split_whitespace().collect::<Vec<_>>().join(" ")
1200}
1201
1202/// What a document declares about itself, in a standard
1203/// vocabulary its format provides — HTML's `<head>` (the `meta`
1204/// name/property pairs, `title`, the `lang` attribute, the
1205/// canonical link, `rel="tag"` links) and a schema.org JSON-LD
1206/// block. Two layers: every declaration lands verbatim under
1207/// `::::name` (`::::description`, `::::og:type`,
1208/// `::::article:section`, `::::schema:@type`), so nothing a page
1209/// says is lost; and the standard vocabularies feed a curated
1210/// core — the tags (`keywords`, `article:tag`, `rel=tag`, JSON-LD
1211/// `keywords`) become traits on the document and `::tags`, the
1212/// one classification (`article:section` / `articleSection`)
1213/// becomes `::category` and a trait too, `description`, `author`,
1214/// and the `published` / `modified` instants. Declared identity
1215/// only: a format adapter reads a cited vocabulary, never a class
1216/// name or a layout.
1217#[derive(Debug, Clone, Default)]
1218pub struct HeadMeta {
1219    /// Every declaration, name → values, in document order; a
1220    /// repeated name accumulates.
1221    pub declared: Vec<(String, Vec<String>)>,
1222    pub title: Option<String>,
1223    pub description: Option<String>,
1224    pub author: Option<String>,
1225    /// The one classification (a section of the site, a category).
1226    pub category: Option<String>,
1227    /// The tags, as written, first appearance wins.
1228    pub tags: Vec<String>,
1229    pub published: Option<Value>,
1230    pub modified: Option<Value>,
1231}
1232
1233impl HeadMeta {
1234    /// Record a declaration verbatim (the lossless layer).
1235    pub fn declare(&mut self, name: &str, value: &str) {
1236        let value = normalize_ws(value);
1237        if name.is_empty() || value.is_empty() {
1238            return;
1239        }
1240        match self.declared.iter_mut().find(|(k, _)| k == name) {
1241            Some((_, vs)) => vs.push(value),
1242            None => self.declared.push((name.to_string(), vec![value])),
1243        }
1244    }
1245
1246    /// Add a tag unless already present (case-insensitively).
1247    pub fn tag(&mut self, tag: &str) {
1248        let tag = normalize_ws(tag);
1249        if tag.is_empty() || self.tags.iter().any(|t| t.eq_ignore_ascii_case(&tag)) {
1250            return;
1251        }
1252        self.tags.push(tag);
1253    }
1254
1255    /// A declaration's value: the string when declared once, the
1256    /// list when repeated.
1257    pub fn declared_value(&self, name: &str) -> Option<Value> {
1258        let (_, vs) = self.declared.iter().find(|(k, _)| k == name)?;
1259        Some(match vs.as_slice() {
1260            [one] => Value::Str(one.clone()),
1261            many => Value::list(many.iter().cloned().map(Value::Str).collect()),
1262        })
1263    }
1264
1265    /// The trait spelling of a tag or category: as declared, case
1266    /// kept, with runs of anything a name cannot carry (spaces,
1267    /// slashes, parentheses) collapsed to one dash — `Execution
1268    /// Model` is `<tag:Execution-Model>`, `now()` is `<tag:now>`.
1269    pub fn trait_name(s: &str) -> String {
1270        let mut out = String::new();
1271        let mut dash = false;
1272        for c in s.chars() {
1273            if c.is_alphanumeric() || matches!(c, '-' | '_' | '.' | '+') {
1274                out.push(c);
1275                dash = c == '-';
1276            } else if !dash && !out.is_empty() {
1277                out.push('-');
1278                dash = true;
1279            }
1280        }
1281        while out.ends_with('-') {
1282            out.pop();
1283        }
1284        out
1285    }
1286}
1287
1288/// What the model derives from its nodes once and keeps while
1289/// they do not change: the citation table, the set of prose
1290/// blocks and each note body's callout. Every mutating method
1291/// drops it (see `mutated`).
1292#[derive(Default)]
1293struct Caches {
1294    cite_table: Vec<(NodeId, u32, NodeId)>,
1295    prose_set: std::collections::HashSet<NodeId>,
1296    callout_of_body: std::collections::HashMap<NodeId, NodeId>,
1297}
1298
1299/// A Quarb adapter over a text-level document.
1300pub struct TextModel {
1301    nodes: Vec<Node>,
1302    root: NodeId,
1303    caches: std::cell::OnceCell<Caches>,
1304    /// Every borne name → its bearer (block labels and point
1305    /// anchors, one namespace, first bearer in document order
1306    /// wins) — the landing map for `-->` and for a sibling
1307    /// document's `#fragment`.
1308    onyms: std::collections::HashMap<String, NodeId>,
1309    /// The document's own URL, when the mount knows it.
1310    document_url: Option<url::Url>,
1311    /// The base relative reference targets join against when the
1312    /// document declares one (html's `<base href>`); else the
1313    /// document's own URL.
1314    link_base: Option<url::Url>,
1315    /// Where the document is served from when no absolute URL is
1316    /// known — the mount's path, an archive member's path — the
1317    /// base `::href` links against.
1318    document_path: Option<String>,
1319    /// The citation scheme `::cite` answers under when the
1320    /// document carries several (`?scheme=`); else the first
1321    /// milestone's.
1322    cite_scheme: Option<String>,
1323    /// The corpus reading leaves the note bodies untokenized
1324    /// (`?notes=omit`): the author's words alone, for work over an
1325    /// annotated edition. By default a note is part of the text.
1326    notes_omitted: bool,
1327    /// The spelling table the corpus reading folds through
1328    /// (`?modernize=`): `::lower` and `::modern` read
1329    /// the modern spelling of a historical edition's tokens; `::`
1330    /// stays the edition's own.
1331    orthography: Option<String>,
1332    /// Alias → canonical campus, for bib field lookup: the
1333    /// bibliogramma vocabulary's census (BibLaTeX's field names
1334    /// are its English rows), so `::author` answers as
1335    /// `::auctor` — in any covered language. Set by the koine
1336    /// route, which holds the dialektos.
1337    bib_aliases: std::collections::HashMap<String, String>,
1338    /// What the document declares about itself (see [`HeadMeta`]);
1339    /// answered on the document node.
1340    head: HeadMeta,
1341    /// The corpus reading's positional index: case-folded form →
1342    /// the tokens bearing it, in document order. Built on first
1343    /// use, once the tokens exist; answers `//token[::lower = "x"]`
1344    /// through `descendants_where` without a walk.
1345    token_index: std::sync::OnceLock<std::collections::HashMap<String, Vec<NodeId>>>,
1346    /// The mentions an annotation marked, in document
1347    /// order of their first token; tokens point in by ordinal.
1348    mentions: Vec<Mention>,
1349    /// The corpus reading is on and the tier is virtual
1350    /// ([`tier`]): no block declared its tokens, so sentences and
1351    /// tokens are computed per block on first reach.
1352    tiered: bool,
1353    /// The sentence bonds the virtual tier segments with.
1354    bonds: Option<syndesmos::Syndesmos>,
1355    /// The blocks' tier tables, built on first reach.
1356    tiers: std::cell::RefCell<std::collections::HashMap<usize, std::rc::Rc<tier::Tier>>>,
1357    /// A given segmentation (`?sentences=`): each block's sentence
1358    /// spans, aligned from the file.
1359    given: std::collections::HashMap<usize, Vec<(u32, u32)>>,
1360    /// Document-wide ordinals per prose block (tokens and
1361    /// sentences before it), built on first demand.
1362    ords: std::cell::OnceCell<std::collections::HashMap<usize, (u32, u32)>>,
1363    /// A cast or places table's annotation on a virtual token.
1364    virt_fields: std::collections::HashMap<NodeId, Vec<(String, String)>>,
1365    /// `->character` / `->place` from a virtual token.
1366    virt_character: std::collections::HashMap<NodeId, NodeId>,
1367    virt_place: std::collections::HashMap<NodeId, NodeId>,
1368    /// The annotated tokens, as rows ([`ann`]); their strings
1369    /// interned once per document; the rows of each sentence node.
1370    ann: Vec<ann::AnnRow>,
1371    ann_names: quarb::names::Names,
1372    ann_of: std::collections::HashMap<usize, Vec<u32>>,
1373    ann_deps: std::cell::OnceCell<std::collections::HashMap<u32, Vec<u32>>>,
1374    ann_edeps: std::cell::OnceCell<std::collections::HashMap<u32, Vec<(u32, u32)>>>,
1375}
1376
1377impl TextModel {
1378    /// Assemble the document tree from a adapter's event stream.
1379    ///
1380    /// Iterative throughout (the stream is flat; prose flattening
1381    /// runs over indices), so pathological nesting cannot overflow
1382    /// the call stack. Lenient on malformed streams: a stray
1383    /// `Close` is ignored, unclosed containers close at the end.
1384    pub fn build(blocks: Vec<Block>) -> Self {
1385        let mut nodes = vec![Node::new(Kind::Document, None)];
1386        let root = NodeId(0);
1387        // Innermost-last stack of open *sections* (outline-derived).
1388        let mut sections: Vec<NodeId> = Vec::new();
1389        // Innermost-last stack of open explicit containers.
1390        let mut containers: Vec<NodeId> = Vec::new();
1391        // The block a NoteRef callout attaches to: the last flow
1392        // node created or appended to.
1393        let mut last_flow: Option<NodeId> = None;
1394        // The block a Label names: the most recently opened block
1395        // of any kind — a heading's section, a paragraph, a
1396        // container, a verbatim — the html-id / attached-\label
1397        // parity rule.
1398        let mut last_block: Option<NodeId> = None;
1399
1400        for block in blocks {
1401            match block {
1402                Block::Heading { level, lemma } => {
1403                    let lemma = normalize_ws(&lemma);
1404                    if !containers.is_empty() {
1405                        // Decorative heading inside a container:
1406                        // not sectioning — lower to a paragraph.
1407                        if !lemma.is_empty() {
1408                            let parent = *containers.last().unwrap();
1409                            let id = push(&mut nodes, Kind::Paragraph, parent);
1410                            nodes[id.0 as usize].text = lemma;
1411                        }
1412                        continue;
1413                    }
1414                    while let Some(&open) = sections.last() {
1415                        if nodes[open.0 as usize].level >= Some(level) {
1416                            sections.pop();
1417                        } else {
1418                            break;
1419                        }
1420                    }
1421                    let parent = sections.last().copied().unwrap_or(root);
1422                    let id = push(&mut nodes, Kind::Section, parent);
1423                    let n = &mut nodes[id.0 as usize];
1424                    n.lemma = Some(lemma);
1425                    n.level = Some(level);
1426                    sections.push(id);
1427                    last_flow = Some(id);
1428                    last_block = Some(id);
1429                }
1430                Block::Paragraph { text } => {
1431                    let text = normalize_ws(&text);
1432                    if text.is_empty() {
1433                        continue;
1434                    }
1435                    let parent = cursor(&sections, &containers, root);
1436                    let id = push(&mut nodes, Kind::Paragraph, parent);
1437                    nodes[id.0 as usize].text = text;
1438                    last_flow = Some(id);
1439                    last_block = Some(id);
1440                }
1441                // A speech is a paragraph that knows its speaker.
1442                Block::Dialogue { lemma, text, lit } => {
1443                    let text = normalize_ws(&text);
1444                    if text.is_empty() {
1445                        continue;
1446                    }
1447                    let parent = cursor(&sections, &containers, root);
1448                    let kind = if lit { Kind::Dialogue } else { Kind::Paragraph };
1449                    let id = push(&mut nodes, kind, parent);
1450                    let n = &mut nodes[id.0 as usize];
1451                    n.text = text;
1452                    n.lemma = Some(normalize_ws(&lemma)).filter(|l| !l.is_empty());
1453                    last_flow = Some(id);
1454                    last_block = Some(id);
1455                }
1456                // The literary reading's speech block.
1457                Block::Speech {
1458                    text,
1459                    genoses,
1460                    fields,
1461                } => {
1462                    let text = normalize_ws(&text);
1463                    if text.is_empty() {
1464                        continue;
1465                    }
1466                    let parent = cursor(&sections, &containers, root);
1467                    let id = push(&mut nodes, Kind::Speech, parent);
1468                    let n = &mut nodes[id.0 as usize];
1469                    n.text = text;
1470                    n.genoses = genoses;
1471                    n.fields = fields;
1472                    last_flow = Some(id);
1473                    last_block = Some(id);
1474                }
1475                // An inline span of the last flow block, nested
1476                // under the innermost earlier span that contains it.
1477                Block::Span {
1478                    kind,
1479                    lo,
1480                    hi,
1481                    genoses,
1482                    fields,
1483                } => {
1484                    // The host: the last flow block — or the entry
1485                    // or sense just opened, when the span was met
1486                    // in its headword line (the dictionary
1487                    // vocabulary: a variant beside the headword),
1488                    // whose text is then the lemma.
1489                    let Some((flow, in_lemma)) = inline_host(&nodes, &containers, last_flow) else {
1490                        continue;
1491                    };
1492                    let host = &nodes[flow.0 as usize];
1493                    let source = if in_lemma {
1494                        host.lemma.as_deref().unwrap_or("")
1495                    } else {
1496                        host.text.as_str()
1497                    };
1498                    let text = source
1499                        .get(lo as usize..hi as usize)
1500                        .unwrap_or("")
1501                        .to_string();
1502                    let parent = enclosing_span(&nodes, flow, lo, hi).unwrap_or(flow);
1503                    let id = push(&mut nodes, Kind::Span, parent);
1504                    let n = &mut nodes[id.0 as usize];
1505                    n.span_kind = Some(kind);
1506                    n.at = Some((lo, hi));
1507                    n.text = text;
1508                    n.genoses = genoses;
1509                    n.fields = fields;
1510                }
1511                Block::Hidden {
1512                    kind,
1513                    text,
1514                    mut genoses,
1515                    fields,
1516                } => {
1517                    let Some((flow, in_lemma)) = inline_host(&nodes, &containers, last_flow) else {
1518                        continue;
1519                    };
1520                    let host = &nodes[flow.0 as usize];
1521                    let end = if in_lemma {
1522                        host.lemma.as_deref().map_or(0, str::len) as u32
1523                    } else {
1524                        host.text.len() as u32
1525                    };
1526                    let id = push(&mut nodes, Kind::Span, flow);
1527                    if !genoses.iter().any(|g| g == "hidden") {
1528                        genoses.push("hidden".to_string());
1529                    }
1530                    let n = &mut nodes[id.0 as usize];
1531                    n.span_kind = Some(kind);
1532                    n.at = Some((end, end));
1533                    n.text = normalize_ws(&text);
1534                    n.genoses = genoses;
1535                    n.fields = fields;
1536                }
1537                Block::Character {
1538                    lemma,
1539                    onym,
1540                    text,
1541                    genoses,
1542                } => {
1543                    let parent = cursor(&sections, &containers, root);
1544                    let id = push(&mut nodes, Kind::Character, parent);
1545                    let n = &mut nodes[id.0 as usize];
1546                    n.lemma = Some(normalize_ws(&lemma)).filter(|l| !l.is_empty());
1547                    n.onym = onym;
1548                    n.text = normalize_ws(&text);
1549                    n.genoses = genoses;
1550                    last_block = Some(id);
1551                }
1552                Block::Parsed {
1553                    lo,
1554                    hi,
1555                    onym,
1556                    parsings,
1557                } => {
1558                    let Some(flow) = last_flow else { continue };
1559                    nodes[flow.0 as usize].declared_tokens.push(DeclaredToken {
1560                        lo,
1561                        hi,
1562                        onym,
1563                        parsings,
1564                    });
1565                }
1566                Block::Periodos { lo, hi, id } => {
1567                    let Some(flow) = last_flow else { continue };
1568                    nodes[flow.0 as usize].declared_sentences.push((lo, hi, id));
1569                }
1570                Block::Milestone { scheme, value, at } => {
1571                    let Some(flow) = last_flow else { continue };
1572                    // `u32::MAX`: the milestone stood in a block with
1573                    // no prose of its own — the end of the flow
1574                    // block before it.
1575                    let at = if at == u32::MAX {
1576                        nodes[flow.0 as usize].text.len() as u32
1577                    } else {
1578                        at
1579                    };
1580                    let parent = enclosing_span(&nodes, flow, at, at).unwrap_or(flow);
1581                    let id = push(&mut nodes, Kind::Milestone, parent);
1582                    let n = &mut nodes[id.0 as usize];
1583                    n.at = Some((at, at));
1584                    n.fields = vec![("scheme".to_string(), scheme), ("value".to_string(), value)];
1585                }
1586                Block::Annotate { fields } => {
1587                    if let Some(b) = last_block {
1588                        nodes[b.0 as usize].fields.extend(fields);
1589                    }
1590                }
1591                Block::Wear { genoses } => {
1592                    if let Some(b) = last_block {
1593                        let own = &mut nodes[b.0 as usize].genoses;
1594                        for g in genoses {
1595                            if !own.contains(&g) {
1596                                own.push(g);
1597                            }
1598                        }
1599                    }
1600                }
1601                Block::Text { text } => {
1602                    let text = normalize_ws(&text);
1603                    if text.is_empty() {
1604                        continue;
1605                    }
1606                    match containers.last() {
1607                        Some(&open) => {
1608                            let own = &mut nodes[open.0 as usize].text;
1609                            if !own.is_empty() {
1610                                own.push(' ');
1611                            }
1612                            own.push_str(&text);
1613                            last_flow = Some(open);
1614                        }
1615                        None => {
1616                            let parent = sections.last().copied().unwrap_or(root);
1617                            let id = push(&mut nodes, Kind::Paragraph, parent);
1618                            nodes[id.0 as usize].text = text;
1619                            last_flow = Some(id);
1620                        }
1621                    }
1622                }
1623                Block::Open { kind, lemma } => {
1624                    // A note body opens at the document root —
1625                    // litogramma's canonical document-end
1626                    // placement — whatever else is open.
1627                    let kind = match kind {
1628                        Container::Note {
1629                            onym,
1630                            family,
1631                            margin,
1632                        } => {
1633                            let id = push(&mut nodes, family.kind(), root);
1634                            let n = &mut nodes[id.0 as usize];
1635                            n.onym = Some(onym.trim().to_string()).filter(|o| !o.is_empty());
1636                            n.margin = margin;
1637                            containers.push(id);
1638                            last_flow = Some(id);
1639                            continue;
1640                        }
1641                        other => other,
1642                    };
1643                    let parent = cursor(&sections, &containers, root);
1644                    let (taxis, onym) = match &kind {
1645                        Container::Entry { taxis, onym } => (*taxis, onym.clone()),
1646                        Container::Sense { taxis } => (*taxis, None),
1647                        _ => (None, None),
1648                    };
1649                    let (nkind, start) = match kind {
1650                        Container::Blockquote => (Kind::Blockquote, None),
1651                        Container::UnorderedList => (Kind::UnorderedList, None),
1652                        Container::OrderedList { start } => (Kind::OrderedList, Some(start)),
1653                        Container::Entry { .. } => (Kind::Entry, None),
1654                        Container::Sense { .. } => (Kind::Sense, None),
1655                        Container::Item => (
1656                            match nodes[parent.0 as usize].kind {
1657                                Kind::OrderedList => Kind::OrderedItem,
1658                                _ => Kind::UnorderedItem,
1659                            },
1660                            None,
1661                        ),
1662                        Container::Note { .. } => unreachable!("handled above"),
1663                    };
1664                    let id = push(&mut nodes, nkind, parent);
1665                    last_block = Some(id);
1666                    nodes[id.0 as usize].lemma =
1667                        lemma.map(|l| normalize_ws(&l)).filter(|l| !l.is_empty());
1668                    if let Some(start) = start {
1669                        nodes[id.0 as usize].start = start;
1670                    }
1671                    if taxis.is_some() {
1672                        nodes[id.0 as usize].taxis = taxis;
1673                    }
1674                    // An entry is autonym: the headword registers
1675                    // as the onym a ref (`See Liberalism`) lands on,
1676                    // the first bearer winning among homographs.
1677                    if nkind == Kind::Entry {
1678                        let n = &mut nodes[id.0 as usize];
1679                        n.onym = onym
1680                            .map(|o| normalize_ws(&o))
1681                            .filter(|o| !o.is_empty())
1682                            .or_else(|| n.lemma.clone());
1683                    }
1684                    if nkind == Kind::OrderedItem {
1685                        // `push` already appended this item, so the
1686                        // count includes it.
1687                        let nth = nodes[parent.0 as usize]
1688                            .children
1689                            .iter()
1690                            .filter(|&&c| nodes[c.0 as usize].kind == Kind::OrderedItem)
1691                            .count() as i64;
1692                        let start = nodes[parent.0 as usize].start;
1693                        nodes[id.0 as usize].taxis = Some(start + nth - 1);
1694                    }
1695                    containers.push(id);
1696                }
1697                Block::Close { hypograph } => {
1698                    if let Some(open) = containers.pop() {
1699                        nodes[open.0 as usize].hypograph = hypograph
1700                            .map(|h| normalize_ws(&h))
1701                            .filter(|h| !h.is_empty());
1702                    }
1703                }
1704                // The callout: a `footnote` child of the flow
1705                // block it sits in, `<deixis>`-traited; the edge
1706                // to its body resolves after the stream.
1707                Block::NoteRef {
1708                    onym,
1709                    family,
1710                    margin,
1711                    at,
1712                } => {
1713                    let parent = last_flow.unwrap_or(root);
1714                    // A declared family names the callout now; an
1715                    // undeclared one is settled at resolution from
1716                    // the body it reaches (footnote when dangling).
1717                    let kind = family.map(NoteFamily::kind).unwrap_or(Kind::Footnote);
1718                    let id = push(&mut nodes, kind, parent);
1719                    let n = &mut nodes[id.0 as usize];
1720                    n.deixis = true;
1721                    n.family_open = family.is_none();
1722                    n.margin = margin;
1723                    n.at = at.map(|a| (a, a));
1724                    n.onym = Some(onym.trim().to_string()).filter(|o| !o.is_empty());
1725                }
1726                Block::Ref {
1727                    target,
1728                    text,
1729                    internal,
1730                } => {
1731                    let parent = last_flow.unwrap_or(root);
1732                    let id = push(&mut nodes, Kind::Ref, parent);
1733                    let n = &mut nodes[id.0 as usize];
1734                    n.internal = internal;
1735                    n.target = Some(target.trim().to_string()).filter(|t| !t.is_empty());
1736                    n.text = text.map(|t| normalize_ws(&t)).unwrap_or_default();
1737                }
1738                Block::Cite { target, locator } => {
1739                    let parent = last_flow.unwrap_or(root);
1740                    let id = push(&mut nodes, Kind::Cit, parent);
1741                    let n = &mut nodes[id.0 as usize];
1742                    n.internal = true;
1743                    n.target = Some(target.trim().to_string()).filter(|t| !t.is_empty());
1744                    n.locator = locator
1745                        .map(|l| l.trim().to_string())
1746                        .filter(|l| !l.is_empty());
1747                }
1748                Block::Bib {
1749                    key,
1750                    text,
1751                    fields,
1752                    genus,
1753                } => {
1754                    let parent = cursor(&sections, &containers, root);
1755                    let id = push(&mut nodes, Kind::Bib, parent);
1756                    last_block = Some(id);
1757                    let n = &mut nodes[id.0 as usize];
1758                    n.onym = Some(key.trim().to_string()).filter(|k| !k.is_empty());
1759                    n.text = normalize_ws(&text);
1760                    n.fields = fields
1761                        .into_iter()
1762                        .map(|(k, v)| (k.trim().to_string(), normalize_ws(&v)))
1763                        .filter(|(k, v)| !k.is_empty() && !v.is_empty())
1764                        .collect();
1765                    n.genus = genus
1766                        .map(|g| g.trim().to_string())
1767                        .filter(|g| !g.is_empty());
1768                }
1769                Block::Anchor { onym } => {
1770                    let parent = last_flow.unwrap_or(root);
1771                    let id = push(&mut nodes, Kind::Anchor, parent);
1772                    nodes[id.0 as usize].onym =
1773                        Some(onym.trim().to_string()).filter(|o| !o.is_empty());
1774                }
1775                Block::Label { onym } => {
1776                    let onym = onym.trim().to_string();
1777                    if onym.is_empty() {
1778                        continue;
1779                    }
1780                    // The most recently opened block bears the
1781                    // name — a heading's section, a paragraph, a
1782                    // blockquote: the html-id / attached-\label
1783                    // parity rule. A label before any block, or on
1784                    // a block already named, degrades to a point.
1785                    let free = last_block
1786                        .map(|b| nodes[b.0 as usize].onym.is_none())
1787                        .unwrap_or(false);
1788                    match (last_block, free) {
1789                        (Some(b), true) => {
1790                            nodes[b.0 as usize].onym = Some(onym);
1791                        }
1792                        _ => {
1793                            let parent = last_flow.unwrap_or(root);
1794                            let id = push(&mut nodes, Kind::Anchor, parent);
1795                            nodes[id.0 as usize].onym = Some(onym);
1796                        }
1797                    }
1798                }
1799                Block::IndexMark { term } => {
1800                    let parent = last_flow.unwrap_or(root);
1801                    let id = push(&mut nodes, Kind::IndexMark, parent);
1802                    nodes[id.0 as usize].onym = Some(quarb_term(&term)).filter(|t| !t.is_empty());
1803                }
1804                Block::Verbatim { lang, text } => {
1805                    let parent = cursor(&sections, &containers, root);
1806                    let id = push(&mut nodes, Kind::Verbatim, parent);
1807                    last_block = Some(id);
1808                    let n = &mut nodes[id.0 as usize];
1809                    n.lang = lang.filter(|l| !l.is_empty());
1810                    n.text = text;
1811                }
1812                Block::Table {
1813                    lemma,
1814                    headers,
1815                    rows,
1816                } => {
1817                    let parent = cursor(&sections, &containers, root);
1818                    lower_table(&mut nodes, parent, lemma, headers, rows);
1819                }
1820                Block::Verse {
1821                    lemma,
1822                    strophes,
1823                    hypograph,
1824                } => {
1825                    let parent = cursor(&sections, &containers, root);
1826                    let id = lower_verse(&mut nodes, parent, lemma, strophes, hypograph);
1827                    last_flow = Some(id);
1828                    last_block = Some(id);
1829                }
1830            }
1831        }
1832
1833        resolve_notes(&mut nodes);
1834        let onyms = resolve_refs(&mut nodes);
1835        flatten_prose(&mut nodes);
1836        let mut model = TextModel {
1837            nodes,
1838            root,
1839            caches: Default::default(),
1840            onyms,
1841            document_url: None,
1842            link_base: None,
1843            document_path: None,
1844            cite_scheme: None,
1845            notes_omitted: false,
1846            orthography: None,
1847            bib_aliases: Default::default(),
1848            head: HeadMeta::default(),
1849            token_index: Default::default(),
1850            mentions: Vec::new(),
1851            tiered: false,
1852            bonds: None,
1853            tiers: Default::default(),
1854            given: Default::default(),
1855            ords: Default::default(),
1856            virt_fields: Default::default(),
1857            virt_character: Default::default(),
1858            virt_place: Default::default(),
1859            ann: Vec::new(),
1860            ann_names: Default::default(),
1861            ann_of: Default::default(),
1862            ann_deps: Default::default(),
1863            ann_edeps: Default::default(),
1864        };
1865        model.link_characters();
1866        model
1867    }
1868
1869    /// Register the bib-field alias census (alias → canonical
1870    /// campus), lowercased keys — the friction remover: a field
1871    /// authored or queried under any covered name answers.
1872    pub fn set_bib_aliases(&mut self, map: std::collections::HashMap<String, String>) {
1873        self.mutated();
1874        self.bib_aliases = map;
1875    }
1876
1877    /// Declare the document's own URL — the base a relative ref
1878    /// target joins against when `-->` reaches across documents.
1879    pub fn set_document_url(&mut self, url: &str) {
1880        self.mutated();
1881        self.document_url = url::Url::parse(url).ok();
1882    }
1883
1884    /// Declare the base relative reference targets join against
1885    /// — html's `<base href>` — when it differs from the
1886    /// document's own URL.
1887    pub fn set_link_base(&mut self, url: &str) {
1888        self.mutated();
1889        self.link_base = url::Url::parse(url).ok();
1890    }
1891
1892    /// Declare where the document is served from when it declares
1893    /// no absolute URL of its own: the base `::href` links against
1894    /// (a mount path, an archive member's path).
1895    pub fn set_document_path(&mut self, path: &str) {
1896        self.mutated();
1897        self.document_path = Some(path.to_string());
1898    }
1899
1900    /// The scheme `::cite` reads: the mount's
1901    /// `?scheme=`.
1902    /// Leave the note bodies out of the corpus reading's tier
1903    /// (`?notes=omit`); set before tokenizing.
1904    pub fn omit_notes(&mut self, omit: bool) {
1905        self.mutated();
1906        self.notes_omitted = omit;
1907    }
1908
1909    /// Lower the reference list a document carries as prose
1910    /// (`?references=STANDARD`): every paragraph that reads as a
1911    /// reference under the standard becomes a `bib` node — its
1912    /// text the projection, the fields under the canonical campi,
1913    /// the genus as trait and `::::genus`, `::onym` the list's
1914    /// number or else the responsible party and year — and the
1915    /// citations in the remaining prose become `cit` nodes
1916    /// (bracketed numbers and ranges with their pinpoint, author-
1917    /// year parentheses) resolved against them. A paragraph that
1918    /// does not parse stays prose; no heading is matched. Returns
1919    /// the number of entries lowered; an unknown standard is an
1920    /// error naming the known ones.
1921    pub fn lower_references(&mut self, standard: &str) -> Result<usize, String> {
1922        self.mutated();
1923        if !quarb::bibref::STANDARDS.contains(&standard) {
1924            return Err(format!(
1925                "unknown standard {standard:?} for references= — this build reads {}",
1926                quarb::bibref::STANDARDS.join(", ")
1927            ));
1928        }
1929        // The entries: (node, parsed) in document order.
1930        let mut entries: Vec<(NodeId, quarb::bibref::Reference)> = Vec::new();
1931        for i in 0..self.nodes.len() {
1932            let n = &self.nodes[i];
1933            // A paragraph, or a list item carrying its text itself
1934            // (a numbered reference list is one).
1935            if !matches!(
1936                n.kind,
1937                Kind::Paragraph | Kind::OrderedItem | Kind::UnorderedItem
1938            ) || n.deixis
1939                || self.note_body_of(NodeId(i as u64)).is_some()
1940            {
1941                continue;
1942            }
1943            // A loose list's item carries its paragraph's text
1944            // itself: the item is the entry, the paragraph inside
1945            // it is not a second one.
1946            if n.kind == Kind::Paragraph
1947                && n.parent.is_some_and(|p| {
1948                    matches!(
1949                        self.nodes[p.0 as usize].kind,
1950                        Kind::OrderedItem | Kind::UnorderedItem
1951                    )
1952                })
1953            {
1954                continue;
1955            }
1956            let text = n.prose.clone();
1957            if text.chars().filter(|c| c.is_alphabetic()).count() < 8 {
1958                continue;
1959            }
1960            if let Ok(Some(r)) = quarb::bibref::parse(standard, &text) {
1961                // A reference has a title at least, and either a
1962                // year or a party: prose seldom carries both.
1963                let dated = r.get("year").is_some() || r.get("author").is_some();
1964                if r.get("title").is_some() && dated {
1965                    entries.push((NodeId(i as u64), r));
1966                }
1967            }
1968        }
1969        if entries.is_empty() {
1970            return Ok(0);
1971        }
1972        // Lower each entry in place: the node's kind, onym, fields
1973        // and genus; its text stays as it was.
1974        let mut census: Vec<(String, Option<String>, Option<String>)> = Vec::new();
1975        for (id, r) in &entries {
1976            // The entry's number: written in the text, or the
1977            // ordinal of the list item that carries it (a
1978            // numbered list's numbering is the format's).
1979            let item = if self.nodes[id.0 as usize].kind == Kind::OrderedItem {
1980                Some(*id)
1981            } else {
1982                self.nodes[id.0 as usize]
1983                    .parent
1984                    .filter(|p| self.nodes[p.0 as usize].kind == Kind::OrderedItem)
1985            };
1986            let ordinal = item.and_then(|p| {
1987                let list = self.nodes[p.0 as usize].parent?;
1988                let siblings = &self.nodes[list.0 as usize].children;
1989                let pos = siblings.iter().position(|c| *c == p)?;
1990                Some((self.nodes[list.0 as usize].start + pos as i64).to_string())
1991            });
1992            let onym = r.label.clone().or(ordinal).unwrap_or_else(|| {
1993                let party = r
1994                    .get("author")
1995                    .map(|a| a.split([',', ' ']).next().unwrap_or(a).to_string())
1996                    .unwrap_or_default();
1997                format!("{}{}", party, r.get("year").unwrap_or(""))
1998            });
1999            let n = &mut self.nodes[id.0 as usize];
2000            n.kind = Kind::Bib;
2001            n.onym = Some(onym.clone());
2002            n.fields = r.campi();
2003            n.genus = r.genus.map(str::to_string);
2004            n.text = n.prose.clone();
2005            census.push((
2006                onym,
2007                r.get("author").map(str::to_string),
2008                r.get("year").map(str::to_string),
2009            ));
2010        }
2011        // The citations in the prose that remains.
2012        let entry_ids: std::collections::HashSet<u64> =
2013            entries.iter().map(|(id, _)| id.0).collect();
2014        let mut new_cits: Vec<(NodeId, String, Option<String>)> = Vec::new();
2015        for i in 0..self.nodes.len() {
2016            let n = &self.nodes[i];
2017            if entry_ids.contains(&(i as u64)) || n.deixis {
2018                continue;
2019            }
2020            if !matches!(
2021                n.kind,
2022                Kind::Paragraph
2023                    | Kind::UnorderedItem
2024                    | Kind::OrderedItem
2025                    | Kind::Footnote
2026                    | Kind::Endnote
2027            ) || n.prose.is_empty()
2028            {
2029                continue;
2030            }
2031            for c in quarb::bibref::citations(&n.prose) {
2032                let target = match &c.key {
2033                    quarb::bibref::CitationKey::Number(k) => Some(k.clone()),
2034                    quarb::bibref::CitationKey::AuthorYear { party, year } => {
2035                        quarb::bibref::resolve_author_year(party, year, &census).map(str::to_string)
2036                    }
2037                };
2038                let Some(target) = target else { continue };
2039                new_cits.push((NodeId(i as u64), target, c.locator.clone()));
2040            }
2041        }
2042        for (parent, target, locator) in new_cits {
2043            let id = push(&mut self.nodes, Kind::Cit, parent);
2044            let n = &mut self.nodes[id.0 as usize];
2045            n.internal = true;
2046            n.target = Some(target);
2047            n.locator = locator;
2048        }
2049        // Resolve anew: the new bearers and the new referrers.
2050        for n in self.nodes.iter_mut() {
2051            if n.kind == Kind::Cit {
2052                n.ref_edge = None;
2053                n.dangling = false;
2054            }
2055            n.ref_cites.clear();
2056        }
2057        let onyms = resolve_refs(&mut self.nodes);
2058        self.onyms = onyms;
2059        // `::author` beside `::auctor`: the English names the
2060        // census maps, when no vocabulary census is attached.
2061        if self.bib_aliases.is_empty() {
2062            self.bib_aliases = quarb::bibref::alias_map()
2063                .into_iter()
2064                .map(|(canon, en)| (en.to_string(), canon.to_string()))
2065                .collect();
2066        }
2067        Ok(entries.len())
2068    }
2069
2070    pub fn set_cite_scheme(&mut self, scheme: &str) {
2071        self.mutated();
2072        self.cite_scheme = Some(scheme.to_string());
2073    }
2074
2075    /// The spelling table the tokens fold through:
2076    /// the mount's `?modernize=`. An unknown table is an error.
2077    pub fn set_orthography(&mut self, table: &str) -> Result<(), String> {
2078        self.mutated();
2079        quarb::translit::modernize("", table)?;
2080        self.orthography = Some(table.to_string());
2081        Ok(())
2082    }
2083
2084    /// A token's modern spelling: the form through the mount's
2085    /// spelling table, else the form itself.
2086    fn modern(&self, form: &str) -> String {
2087        match &self.orthography {
2088            Some(t) => quarb::translit::modernize(form, t).unwrap_or_else(|_| form.to_string()),
2089            None => form.to_string(),
2090        }
2091    }
2092
2093    /// A token's `::lower`: the modern spelling, case-folded.
2094    fn lower(&self, form: &str) -> String {
2095        self.modern(form).to_lowercase()
2096    }
2097
2098    /// The document's milestones under the citation scheme, in
2099    /// document order: (flow block, offset, milestone).
2100    fn cite_table(&self) -> Vec<(NodeId, u32, NodeId)> {
2101        let mut scheme = self.cite_scheme.clone();
2102        let mut out = Vec::new();
2103        for i in 0..self.nodes.len() {
2104            let n = &self.nodes[i];
2105            if n.kind != Kind::Milestone {
2106                continue;
2107            }
2108            let this = n
2109                .fields
2110                .iter()
2111                .find(|(k, _)| k == "scheme")
2112                .map(|(_, v)| v.clone());
2113            let Some(this) = this else { continue };
2114            match &scheme {
2115                None => scheme = Some(this),
2116                Some(s) if *s != this => continue,
2117                _ => {}
2118            }
2119            let at = n.at.map(|(a, _)| a).unwrap_or(0);
2120            let mut block = NodeId(i as u64);
2121            while self.nodes[block.0 as usize].kind == Kind::Span
2122                || self.nodes[block.0 as usize].kind == Kind::Milestone
2123            {
2124                match self.nodes[block.0 as usize].parent {
2125                    Some(p) => block = p,
2126                    None => break,
2127                }
2128            }
2129            out.push((block, at, NodeId(i as u64)));
2130        }
2131        out.sort_by_key(|(b, at, _)| (b.0, *at));
2132        out
2133    }
2134
2135    /// The citation in force at a node: the value of
2136    /// the last milestone under the citation scheme at or before
2137    /// the node's point in the document — a token's or a span's
2138    /// offset in its block, a sentence's first token, a flow
2139    /// block's start; a container (a section, the document) cites
2140    /// as the first milestone inside it. Null where no milestone
2141    /// governs.
2142    /// The nodes changed: whatever was derived from them is stale.
2143    fn mutated(&mut self) {
2144        self.caches = Default::default();
2145        self.ords = Default::default();
2146        self.ann_deps = Default::default();
2147        self.ann_edeps = Default::default();
2148    }
2149
2150    /// The derived tables, built on first use.
2151    fn caches(&self) -> &Caches {
2152        self.caches.get_or_init(|| {
2153            let mut callout_of_body = std::collections::HashMap::new();
2154            for (i, n) in self.nodes.iter().enumerate() {
2155                if n.deixis
2156                    && let Some(body) = n.note_edge
2157                {
2158                    callout_of_body.entry(body).or_insert(NodeId(i as u64));
2159                }
2160            }
2161            Caches {
2162                cite_table: self.cite_table(),
2163                prose_set: self.prose_blocks().into_iter().collect(),
2164                callout_of_body,
2165            }
2166        })
2167    }
2168
2169    fn cite(&self, node: NodeId) -> Option<String> {
2170        let caches = self.caches();
2171        let table = &caches.cite_table;
2172        if table.is_empty() {
2173            return self.structural_cite(node);
2174        }
2175        let n = &self.nodes[node.0 as usize];
2176        let value = |m: NodeId| {
2177            self.nodes[m.0 as usize]
2178                .fields
2179                .iter()
2180                .find(|(k, _)| k == "value")
2181                .map(|(_, v)| v.clone())
2182        };
2183        // Containers: the first milestone beneath.
2184        let container = matches!(n.kind, Kind::Document | Kind::Section)
2185            || (n.kind != Kind::Span
2186                && n.kind != Kind::Milestone
2187                && n.kind != Kind::Sentence
2188                && !caches.prose_set.contains(&node)
2189                && !n.children.is_empty()
2190                && n.prose.is_empty());
2191        if container {
2192            let inside = |mut b: NodeId| {
2193                loop {
2194                    if b == node {
2195                        return true;
2196                    }
2197                    match self.nodes[b.0 as usize].parent {
2198                        Some(p) => b = p,
2199                        None => return false,
2200                    }
2201                }
2202            };
2203            return table
2204                .iter()
2205                .find(|(b, _, _)| inside(*b))
2206                .and_then(|(_, _, m)| value(*m));
2207        }
2208        // A note body, and everything in it, cites as its callout.
2209        let body = if matches!(n.kind, Kind::Footnote | Kind::Endnote | Kind::Aside) && !n.deixis {
2210            Some(node)
2211        } else {
2212            self.note_body_of(node)
2213        };
2214        if let Some(body) = body {
2215            let callout = caches.callout_of_body.get(&body).copied();
2216            return callout.and_then(|c| self.cite(c));
2217        }
2218        // Everything else: its point in the document.
2219        let (block, at) = match n.kind {
2220            _ if n.deixis && n.at.is_some() => {
2221                (n.parent.unwrap_or(node), n.at.map(|(a, _)| a).unwrap_or(0))
2222            }
2223            Kind::Milestone | Kind::Span => {
2224                let mut b = node;
2225                while matches!(self.nodes[b.0 as usize].kind, Kind::Span | Kind::Milestone) {
2226                    match self.nodes[b.0 as usize].parent {
2227                        Some(p) => b = p,
2228                        None => break,
2229                    }
2230                }
2231                (b, n.at.map(|(a, _)| a).unwrap_or(0))
2232            }
2233            Kind::Sentence if n.derived => {
2234                let at = self
2235                    .ann_of
2236                    .get(&(node.0 as usize))
2237                    .and_then(|rows| rows.first())
2238                    .map(|&r| self.ann[r as usize].at)
2239                    .unwrap_or(0);
2240                (n.parent.unwrap_or(node), at)
2241            }
2242            _ => (node, 0),
2243        };
2244        let _ = value;
2245        self.cite_at(block, at, node)
2246    }
2247
2248    /// The citation in force at byte `at` of `block`: the last
2249    /// milestone at or before it, else the structural citation of
2250    /// `fallback`.
2251    fn cite_at(&self, block: NodeId, at: u32, fallback: NodeId) -> Option<String> {
2252        let caches = self.caches();
2253        let value = |m: NodeId| {
2254            self.nodes[m.0 as usize]
2255                .fields
2256                .iter()
2257                .find(|(k, _)| k == "value")
2258                .map(|(_, v)| v.clone())
2259        };
2260        caches
2261            .cite_table
2262            .iter()
2263            .rev()
2264            .find(|(b, a, _)| (b.0, *a) <= (block.0, at))
2265            .and_then(|(_, _, m)| value(*m))
2266            .or_else(|| self.structural_cite(fallback))
2267    }
2268
2269    /// The citation a text without milestones still has: the
2270    /// enclosing sections' names from the outermost in, joined by
2271    /// " / " — a chapter, an act and its scene. Null at the root.
2272    fn structural_cite(&self, node: NodeId) -> Option<String> {
2273        let mut names: Vec<String> = Vec::new();
2274        let mut cur = Some(node);
2275        while let Some(n) = cur {
2276            let node = &self.nodes[n.0 as usize];
2277            if node.kind == Kind::Section
2278                && let Some(l) = &node.lemma
2279                && !l.is_empty()
2280            {
2281                names.push(l.clone());
2282            }
2283            cur = node.parent;
2284        }
2285        if names.is_empty() {
2286            return None;
2287        }
2288        names.reverse();
2289        Some(names.join(" / "))
2290    }
2291
2292    /// The base `::href` links against: the document's URL, else
2293    /// its declared path.
2294    fn href_base(&self) -> Option<String> {
2295        if let Some(u) = &self.document_url {
2296            return Some(u.to_string());
2297        }
2298        self.document_path.clone()
2299    }
2300
2301    /// The nearest bearer of a name, the node itself first, then
2302    /// its ancestors — the anchor a link into this node lands on.
2303    /// `exact` says the node bears it itself.
2304    fn nearest_onym(&self, node: NodeId) -> Option<(String, bool)> {
2305        let mut cur = Some(node);
2306        let mut exact = true;
2307        while let Some(id) = cur {
2308            let n = &self.nodes[id.0 as usize];
2309            if n.kind != Kind::Document
2310                && !n.deixis
2311                && let Some(o) = &n.onym
2312            {
2313                return Some((o.clone(), exact));
2314            }
2315            exact = false;
2316            cur = n.parent;
2317        }
2318        None
2319    }
2320
2321    /// A link into `node`, as far as the served form can narrow
2322    /// it. The base is the document's URL (its canonical link,
2323    /// else the mount's path). An HTML page — a base ending in
2324    /// `.html` / `.htm` / `/` or without an extension — narrows
2325    /// with the nearest anchor and, for a block below the anchored
2326    /// one, a text fragment (`#:~:text=start,end`) built from the
2327    /// node's own prose. A PDF narrows to nothing yet; any other
2328    /// form links the document alone, since no browser takes a
2329    /// fragment into it. The narrowing stays available as data
2330    /// beside the link (`::anchor`, the prose) for a client that
2331    /// renders the document itself.
2332    fn href(&self, node: NodeId) -> Option<String> {
2333        let base = self.href_base()?;
2334        let n = &self.nodes[node.0 as usize];
2335        if n.kind == Kind::Document {
2336            return Some(base);
2337        }
2338        let served = base.split(['?', '#']).next().unwrap_or(&base);
2339        let last = served.rsplit('/').next().unwrap_or(served);
2340        let html = served.ends_with('/')
2341            || !last.contains('.')
2342            || last.ends_with(".html")
2343            || last.ends_with(".htm");
2344        if !html {
2345            return Some(base);
2346        }
2347        let mut out = base;
2348        let (anchor, exact) = match self.nearest_onym(node) {
2349            Some((a, exact)) => (Some(a), exact),
2350            None => (None, false),
2351        };
2352        if let Some(a) = &anchor {
2353            out.push('#');
2354            out.push_str(a);
2355        }
2356        if !exact {
2357            let words: Vec<&str> = n.prose.split_whitespace().collect();
2358            if !words.is_empty() {
2359                use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};
2360                let enc =
2361                    |ws: &[&str]| utf8_percent_encode(&ws.join(" "), NON_ALPHANUMERIC).to_string();
2362                if anchor.is_none() {
2363                    out.push('#');
2364                }
2365                out.push_str(":~:text=");
2366                if words.len() <= 12 {
2367                    out.push_str(&enc(&words));
2368                } else {
2369                    out.push_str(&enc(&words[..6]));
2370                    out.push(',');
2371                    out.push_str(&enc(&words[words.len() - 6..]));
2372                }
2373            }
2374        }
2375        Some(out)
2376    }
2377
2378    /// The document's title for a link: what its head declares,
2379    /// else its first top-level section's lemma.
2380    fn document_title(&self) -> Option<String> {
2381        if let Some(t) = &self.head.title {
2382            return Some(t.clone());
2383        }
2384        self.nodes[self.root.0 as usize]
2385            .children
2386            .iter()
2387            .map(|c| &self.nodes[c.0 as usize])
2388            .find(|n| n.kind == Kind::Section)
2389            .and_then(|n| n.lemma.clone())
2390    }
2391
2392    /// Attach what the document declares about itself; the format
2393    /// adapter reads it from the source's own metadata vocabulary.
2394    pub fn set_head_meta(&mut self, head: HeadMeta) {
2395        self.mutated();
2396        self.head = head;
2397    }
2398
2399    /// What the document declares about itself.
2400    pub fn head_meta(&self) -> &HeadMeta {
2401        &self.head
2402    }
2403
2404    /// The document's text as one string: every section's lemma
2405    /// and every block's prose, one per line, in document order —
2406    /// the lowering a substrate stores beside the document so a
2407    /// substring predicate inside the graft can be prefiltered on
2408    /// it. The invariant a prefilter relies on: every grafted
2409    /// node's prose (and every lemma) is a substring of this.
2410    pub fn plain_text(&self) -> String {
2411        let mut out = String::new();
2412        let mut stack = vec![self.root];
2413        while let Some(id) = stack.pop() {
2414            let n = &self.nodes[id.0 as usize];
2415            if let Some(l) = &n.lemma
2416                && !l.is_empty()
2417            {
2418                out.push_str(l);
2419                out.push('\n');
2420            }
2421            // Every node's own prose, leaf or not: a paragraph that
2422            // holds a ref or an anchor still has prose of its own,
2423            // and the invariant covers it too.
2424            if !n.prose.is_empty() && n.kind != Kind::Document {
2425                out.push_str(&n.prose);
2426                out.push('\n');
2427            }
2428            for c in n.children.iter().rev() {
2429                stack.push(*c);
2430            }
2431        }
2432        out
2433    }
2434
2435    /// How many nodes the document holds (the root included) — the
2436    /// bound a host needs to pack this document's node ids into a
2437    /// wider id space.
2438    pub fn node_count(&self) -> usize {
2439        self.nodes.len()
2440    }
2441
2442    /// Read plain text: blank-line-separated paragraphs, each
2443    /// collapsed to one line — the atramento paragraph rule. No
2444    /// headings, no markup.
2445    pub fn parse_plain(text: &str) -> Self {
2446        let mut blocks = Vec::new();
2447        let mut para: Vec<&str> = Vec::new();
2448        for line in text.lines() {
2449            if line.trim().is_empty() {
2450                if !para.is_empty() {
2451                    blocks.push(Block::Paragraph {
2452                        text: para.join(" "),
2453                    });
2454                    para.clear();
2455                }
2456            } else {
2457                para.push(line);
2458            }
2459        }
2460        if !para.is_empty() {
2461            blocks.push(Block::Paragraph {
2462                text: para.join(" "),
2463            });
2464        }
2465        Self::build(blocks)
2466    }
2467
2468    /// A locator path to `node`, like `/section[2]/paragraph[3]`,
2469    /// for rendering. A `[n]` index is added only to disambiguate
2470    /// same-name siblings.
2471    pub fn locator(&self, node: NodeId) -> String {
2472        if let Some(v) = tier::virt(node) {
2473            return self.v_locator(v);
2474        }
2475        let mut segments = Vec::new();
2476        let mut cur = Some(node);
2477        while let Some(id) = cur {
2478            let n = &self.nodes[id.0 as usize];
2479            if let Some(name) = n.kind.name() {
2480                segments.push(self.segment(id, name));
2481            }
2482            cur = n.parent;
2483        }
2484        segments.reverse();
2485        format!("/{}", segments.join("/"))
2486    }
2487
2488    fn segment(&self, node: NodeId, name: &str) -> String {
2489        let Some(parent) = self.nodes[node.0 as usize].parent else {
2490            return name.to_string();
2491        };
2492        let siblings = &self.nodes[parent.0 as usize].children;
2493        let same_name: Vec<NodeId> = siblings
2494            .iter()
2495            .copied()
2496            .filter(|&s| self.nodes[s.0 as usize].kind == self.nodes[node.0 as usize].kind)
2497            .collect();
2498        if same_name.len() > 1 {
2499            let n = same_name.iter().position(|&s| s == node).unwrap() + 1;
2500            format!("{name}[{n}]")
2501        } else {
2502            name.to_string()
2503        }
2504    }
2505}
2506
2507/// Where the next block lands: the innermost open container, else
2508/// the innermost open section, else the root.
2509/// Ruling #35: wire each callout to its body by (family, onym) —
2510/// first body wins on a duplicate — build the reverse index, and
2511/// mark the dangling: a callout with no body keeps its node,
2512/// carries `<dangling>` and `::::resolved = false`, and emits no
2513/// edge.
2514fn resolve_notes(nodes: &mut [Node]) {
2515    let mut bodies: std::collections::HashMap<(Kind, String), NodeId> = Default::default();
2516    for (i, n) in nodes.iter().enumerate() {
2517        if matches!(n.kind, Kind::Footnote | Kind::Endnote | Kind::Aside)
2518            && !n.deixis
2519            && let Some(o) = &n.onym
2520        {
2521            bodies
2522                .entry((n.kind, o.clone()))
2523                .or_insert(NodeId(i as u64));
2524        }
2525    }
2526    for i in 0..nodes.len() {
2527        if !matches!(nodes[i].kind, Kind::Footnote | Kind::Endnote | Kind::Aside)
2528            || !nodes[i].deixis
2529        {
2530            continue;
2531        }
2532        let Some(onym) = nodes[i].onym.clone() else {
2533            nodes[i].dangling = true;
2534            continue;
2535        };
2536        let hit = if nodes[i].family_open {
2537            // No declared family (an HTML noteref): the body's
2538            // family is the callout's; footnote when dangling.
2539            [Kind::Footnote, Kind::Endnote, Kind::Aside]
2540                .into_iter()
2541                .find_map(|k| bodies.get(&(k, onym.clone())).map(|b| (k, *b)))
2542        } else {
2543            bodies
2544                .get(&(nodes[i].kind, onym))
2545                .map(|b| (nodes[i].kind, *b))
2546        };
2547        match hit {
2548            Some((family, body)) => {
2549                nodes[i].kind = family;
2550                nodes[i].note_edge = Some(body);
2551                let callout = NodeId(i as u64);
2552                nodes[body.0 as usize].cites.push(callout);
2553            }
2554            None => nodes[i].dangling = true,
2555        }
2556    }
2557}
2558
2559/// An index term as written, with the `|...` formatting and range
2560/// directives stripped (ranges and
2561/// see-references are recorded follow-ups).
2562fn quarb_term(term: &str) -> String {
2563    normalize_ws(term.split('|').next().unwrap_or(term))
2564}
2565
2566fn cursor(sections: &[NodeId], containers: &[NodeId], root: NodeId) -> NodeId {
2567    containers
2568        .last()
2569        .or(sections.last())
2570        .copied()
2571        .unwrap_or(root)
2572}
2573
2574impl TextModel {
2575    /// The corpus reading: give every prose block its
2576    /// tokens as children. Segmentation is UAX #29 word boundaries
2577    /// (the standard `sentences` already follows): `don’t`, `Tom’s`
2578    /// and `3,000` are one token each, an em dash is its own, and
2579    /// whitespace is none. Positions and sentence ordinals count
2580    /// through the document in reading order. The block's own
2581    /// `::` is unchanged; the tokens sit after any children it
2582    /// already had (an inline ref, a callout).
2583    /// The literary reading's cast layer: declare the
2584    /// characters a cast table names (those the source did not),
2585    /// annotate every word token a row's pattern matches with the
2586    /// row's id as its `prosopon`, then resolve every mention to
2587    /// its character.
2588    pub fn apply_cast(&mut self, rows: &[CastRow]) {
2589        self.mutated();
2590        for row in rows {
2591            let known = self
2592                .nodes
2593                .iter()
2594                .any(|n| n.kind == Kind::Character && n.onym.as_deref() == Some(row.id.as_str()));
2595            if known {
2596                continue;
2597            }
2598            let root = NodeId(0);
2599            let id = push(&mut self.nodes, Kind::Character, root);
2600            let n = &mut self.nodes[id.0 as usize];
2601            n.lemma = Some(row.name.clone()).filter(|l| !l.is_empty());
2602            n.onym = Some(row.id.clone());
2603            n.fields = row.fields.clone();
2604        }
2605        // A pattern of several words ("Aunt Polly") matches a run
2606        // of word tokens under one parent, joined by a space; the
2607        // run's first token carries the mention.
2608        let patterns: Vec<(&CastRow, &regex::Regex, usize)> = rows
2609            .iter()
2610            .filter_map(|r| r.pattern.as_ref().map(|p| (r, p, r.words.max(1))))
2611            .collect();
2612        let widest = patterns.iter().map(|(_, _, w)| *w).max().unwrap_or(0);
2613        if widest > 0 {
2614            let words = self.word_runs();
2615            for (k, (id, _, parent)) in words.iter().enumerate() {
2616                if self.has_token_field(*id, "prosopon") {
2617                    continue;
2618                }
2619                let mut run = vec![words[k].1.as_str()];
2620                for (_, form, p) in words.iter().skip(k + 1).take(widest - 1) {
2621                    if p != parent {
2622                        break;
2623                    }
2624                    run.push(form.as_str());
2625                }
2626                let hit = patterns.iter().find(|(_, p, w)| {
2627                    (1..=(*w).min(run.len()))
2628                        .rev()
2629                        .any(|n| p.is_match(&run[..n].join(" ")))
2630                });
2631                if let Some((row, _, _)) = hit {
2632                    self.push_token_field(*id, "prosopon", row.id.clone());
2633                }
2634            }
2635        }
2636        // A coreference chain (a `cluster` column, the source's
2637        // chain id): every mention of the chain is the character's,
2638        // its first token carrying the mention.
2639        let chains: Vec<(&CastRow, &str)> = rows
2640            .iter()
2641            .filter_map(|r| {
2642                r.fields
2643                    .iter()
2644                    .find(|(k, _)| k == "cluster" || k == "chain")
2645                    .map(|(_, v)| (r, v.as_str()))
2646            })
2647            .collect();
2648        if !chains.is_empty() {
2649            let firsts: Vec<(NodeId, String)> = self
2650                .mentions
2651                .iter()
2652                .filter_map(|m| {
2653                    let c = m.cluster.as_deref()?;
2654                    let (row, _) = chains.iter().find(|(_, id)| *id == c)?;
2655                    Some((*m.tokens.first()?, row.id.clone()))
2656                })
2657                .collect();
2658            for (t, id) in firsts {
2659                self.push_token_field(t, "prosopon", id);
2660            }
2661        }
2662        // A play's speech names its speaker by the label the source
2663        // printed (its lemma): a label a pattern matches is that
2664        // character's speech.
2665        if !patterns.is_empty() {
2666            for i in 0..self.nodes.len() {
2667                let n = &self.nodes[i];
2668                if n.kind != Kind::Dialogue || n.fields.iter().any(|(k, _)| k == "prosopon") {
2669                    continue;
2670                }
2671                let Some(label) = n.lemma.clone() else {
2672                    continue;
2673                };
2674                if let Some((row, _, _)) = patterns.iter().find(|(_, p, _)| p.is_match(&label)) {
2675                    self.nodes[i]
2676                        .fields
2677                        .push(("prosopon".to_string(), row.id.clone()));
2678                }
2679            }
2680        }
2681        self.link_characters();
2682    }
2683
2684    /// Every word token in document order with its form and its
2685    /// parent (the stored tier's nodes, or the virtual tier's):
2686    /// what a cast or places table's patterns run over.
2687    fn word_runs(&self) -> Vec<(NodeId, String, NodeId)> {
2688        if self.tiered {
2689            return self.tier_words();
2690        }
2691        self.ann_words()
2692    }
2693
2694    /// Resolve `->character`: every node whose own `prosopon`
2695    /// names a declared character links to it, and the character
2696    /// lists it among its mentions.
2697    fn link_characters(&mut self) {
2698        self.mutated();
2699        let mut by_onym: std::collections::HashMap<String, NodeId> =
2700            std::collections::HashMap::new();
2701        for (i, n) in self.nodes.iter().enumerate() {
2702            if n.kind == Kind::Character
2703                && let Some(o) = &n.onym
2704            {
2705                by_onym.entry(o.clone()).or_insert(NodeId(i as u64));
2706            }
2707        }
2708        for n in &mut self.nodes {
2709            n.mentions.clear();
2710        }
2711        for i in 0..self.nodes.len() {
2712            if self.nodes[i].kind == Kind::Character {
2713                continue;
2714            }
2715            let target = self.nodes[i]
2716                .fields
2717                .iter()
2718                .find(|(k, _)| k == "prosopon")
2719                .and_then(|(_, v)| by_onym.get(v).copied());
2720            self.nodes[i].character = target;
2721            if let Some(c) = target {
2722                self.nodes[c.0 as usize].mentions.push(NodeId(i as u64));
2723            }
2724        }
2725        // The virtual tier's mentions, after the stored nodes' as
2726        // the stored tokens came after every block.
2727        self.virt_character.clear();
2728        let mut virt: Vec<(NodeId, NodeId)> = self
2729            .virt_fields
2730            .iter()
2731            .filter_map(|(id, f)| {
2732                let v = f.iter().find(|(k, _)| k == "prosopon")?;
2733                Some((*id, *by_onym.get(&v.1)?))
2734            })
2735            .collect();
2736        virt.sort();
2737        for (id, c) in virt {
2738            self.virt_character.insert(id, c);
2739            self.nodes[c.0 as usize].mentions.push(id);
2740        }
2741    }
2742
2743    /// The literary reading's gazetteer: a places
2744    /// table declares the places a source names, as a cast table
2745    /// declares its people. Each row is a `place` node under the
2746    /// root (`::lemma` its name, `::onym` its id, every other
2747    /// column a field — `::lat`, `::lon`, `::kind`); a word token a
2748    /// row's pattern matches answers the row's id as `::chora` and
2749    /// reaches the place as `->place`, and the place reaches its
2750    /// mentions back as `<-place`.
2751    pub fn apply_places(&mut self, rows: &[CastRow]) {
2752        self.mutated();
2753        for row in rows {
2754            let known = self
2755                .nodes
2756                .iter()
2757                .any(|n| n.kind == Kind::Place && n.onym.as_deref() == Some(row.id.as_str()));
2758            if known {
2759                continue;
2760            }
2761            let root = NodeId(0);
2762            let id = push(&mut self.nodes, Kind::Place, root);
2763            let n = &mut self.nodes[id.0 as usize];
2764            n.lemma = Some(row.name.clone()).filter(|l| !l.is_empty());
2765            n.onym = Some(row.id.clone());
2766            n.fields = row.fields.clone();
2767        }
2768        let patterns: Vec<(&CastRow, &regex::Regex, usize)> = rows
2769            .iter()
2770            .filter_map(|r| r.pattern.as_ref().map(|p| (r, p, r.words.max(1))))
2771            .collect();
2772        let widest = patterns.iter().map(|(_, _, w)| *w).max().unwrap_or(0);
2773        if widest > 0 {
2774            let words = self.word_runs();
2775            for (k, (id, _, parent)) in words.iter().enumerate() {
2776                if self.has_token_field(*id, "chora") {
2777                    continue;
2778                }
2779                let mut run = vec![words[k].1.as_str()];
2780                for (_, form, p) in words.iter().skip(k + 1).take(widest - 1) {
2781                    if p != parent {
2782                        break;
2783                    }
2784                    run.push(form.as_str());
2785                }
2786                let hit = patterns.iter().find(|(_, p, w)| {
2787                    (1..=(*w).min(run.len()))
2788                        .rev()
2789                        .any(|n| p.is_match(&run[..n].join(" ")))
2790                });
2791                if let Some((row, _, _)) = hit {
2792                    self.push_token_field(*id, "chora", row.id.clone());
2793                }
2794            }
2795        }
2796        self.link_places();
2797    }
2798
2799    /// Resolve `->place`: every node whose own `chora` names a
2800    /// declared place links to it, and the place lists it among
2801    /// its mentions.
2802    fn link_places(&mut self) {
2803        self.mutated();
2804        let mut by_onym: std::collections::HashMap<String, NodeId> =
2805            std::collections::HashMap::new();
2806        for (i, n) in self.nodes.iter().enumerate() {
2807            if n.kind == Kind::Place
2808                && let Some(o) = &n.onym
2809            {
2810                by_onym.entry(o.clone()).or_insert(NodeId(i as u64));
2811            }
2812        }
2813        for n in &mut self.nodes {
2814            if n.kind == Kind::Place {
2815                n.mentions.clear();
2816            }
2817        }
2818        for i in 0..self.nodes.len() {
2819            if self.nodes[i].kind == Kind::Place {
2820                continue;
2821            }
2822            let target = self.nodes[i]
2823                .fields
2824                .iter()
2825                .find(|(k, _)| k == "chora")
2826                .and_then(|(_, v)| by_onym.get(v).copied());
2827            self.nodes[i].place = target;
2828            if let Some(c) = target {
2829                self.nodes[c.0 as usize].mentions.push(NodeId(i as u64));
2830            }
2831        }
2832        self.virt_place.clear();
2833        let mut virt: Vec<(NodeId, NodeId)> = self
2834            .virt_fields
2835            .iter()
2836            .filter_map(|(id, f)| {
2837                let v = f.iter().find(|(k, _)| k == "chora")?;
2838                Some((*id, *by_onym.get(&v.1)?))
2839            })
2840            .collect();
2841        virt.sort();
2842        for (id, c) in virt {
2843            self.virt_place.insert(id, c);
2844            self.nodes[c.0 as usize].mentions.push(id);
2845        }
2846    }
2847
2848    pub fn tokenize(&mut self) {
2849        self.mutated();
2850        self.tokenize_with(None)
2851    }
2852
2853    /// Whether `id` is a block the corpus reading tokenizes — a
2854    /// paragraph, item, verse line, speech, or a note body that
2855    /// carries its own prose.
2856    fn is_prose_block(&self, id: usize) -> bool {
2857        let node = &self.nodes[id];
2858        let prose_block = match node.kind {
2859            // A treebank paragraph delegates to its declared
2860            // sentences; the sentences the
2861            // corpus reading derives beneath a block (ruling
2862            // #64) are annotation, not blocks.
2863            Kind::Paragraph => !node.children.iter().any(|c| {
2864                let c = &self.nodes[c.0 as usize];
2865                c.kind == Kind::Sentence && !c.derived
2866            }),
2867            Kind::Sentence => !node.derived,
2868            Kind::UnorderedItem | Kind::OrderedItem | Kind::Stichos => true,
2869            Kind::Speech | Kind::Dialogue => true,
2870            // A body is a block of its own only when it holds its
2871            // text directly — a body of paragraphs carries their
2872            // text too, and the paragraphs are the blocks.
2873            Kind::Footnote | Kind::Endnote | Kind::Aside => {
2874                !node.deixis && !self.holds_prose_block(id)
2875            }
2876            _ => false,
2877        };
2878        prose_block && !node.prose.is_empty()
2879    }
2880
2881    /// Whether a prose block stands anywhere beneath `id`.
2882    fn holds_prose_block(&self, id: usize) -> bool {
2883        self.nodes[id].children.iter().any(|c| {
2884            let c = c.0 as usize;
2885            self.is_prose_block(c) || self.holds_prose_block(c)
2886        })
2887    }
2888
2889    /// The prose blocks the corpus reading tokenizes, in reading
2890    /// order — the order the text is printed in. A note is part of
2891    /// the text (spec: the corpus reading; the user's ruling of
2892    /// 2026-10-07): a footnote's or an aside's body follows the
2893    /// block that holds its callout, as at the foot of that page;
2894    /// an endnote's body, and a body no callout reaches, follow
2895    /// the flow, as at the end. The tree is untouched — bodies
2896    /// stay under their note node, reached by `->footnote` — only
2897    /// `::::n` and the sentence tier follow the printed order, so
2898    /// a window or a dispersion does not run out of the prose into
2899    /// a commentary. With `?notes=omit` the bodies are left
2900    /// untokenized.
2901    fn prose_blocks(&self) -> Vec<NodeId> {
2902        let mut flow = Vec::new();
2903        let mut by_body: std::collections::BTreeMap<NodeId, Vec<NodeId>> = Default::default();
2904        for id in 0..self.nodes.len() {
2905            if !self.is_prose_block(id) {
2906                continue;
2907            }
2908            let block = NodeId(id as u64);
2909            match self.in_body(block) {
2910                Some(body) => by_body.entry(body).or_default().push(block),
2911                None => flow.push(block),
2912            }
2913        }
2914        if self.notes_omitted || by_body.is_empty() {
2915            return flow;
2916        }
2917        // Each body after the block its first callout stands in —
2918        // the nearest enclosing prose block, inside another body
2919        // when a note is itself noted — or after the flow.
2920        let mut after: std::collections::HashMap<NodeId, Vec<NodeId>> = Default::default();
2921        let mut trailing = Vec::new();
2922        for &body in by_body.keys() {
2923            let n = &self.nodes[body.0 as usize];
2924            let anchor = (n.kind != Kind::Endnote)
2925                .then(|| n.cites.first().and_then(|&c| self.block_of(c)))
2926                .flatten();
2927            match anchor {
2928                Some(block) => after.entry(block).or_default().push(body),
2929                None => trailing.push(body),
2930            }
2931        }
2932        let mut out = Vec::with_capacity(self.nodes.len());
2933        let mut seen: std::collections::HashSet<NodeId> = Default::default();
2934        fn emit(
2935            blocks: &[NodeId],
2936            by_body: &std::collections::BTreeMap<NodeId, Vec<NodeId>>,
2937            after: &std::collections::HashMap<NodeId, Vec<NodeId>>,
2938            seen: &mut std::collections::HashSet<NodeId>,
2939            out: &mut Vec<NodeId>,
2940        ) {
2941            for &b in blocks {
2942                out.push(b);
2943                if let Some(bodies) = after.get(&b) {
2944                    for &body in bodies {
2945                        if seen.insert(body) {
2946                            emit(&by_body[&body], by_body, after, seen, out);
2947                        }
2948                    }
2949                }
2950            }
2951        }
2952        emit(&flow, &by_body, &after, &mut seen, &mut out);
2953        for body in trailing {
2954            if seen.insert(body) {
2955                emit(&by_body[&body], &by_body, &after, &mut seen, &mut out);
2956            }
2957        }
2958        // A body whose callout block was never reached (the callout
2959        // stands outside the prose, or in a cycle): after the flow.
2960        for &body in by_body.keys() {
2961            if seen.insert(body) {
2962                emit(&by_body[&body], &by_body, &after, &mut seen, &mut out);
2963            }
2964        }
2965        out
2966    }
2967
2968    /// The note body `node` stands in, if any — the body itself
2969    /// counts, for a body that carries its own prose.
2970    fn in_body(&self, node: NodeId) -> Option<NodeId> {
2971        let n = &self.nodes[node.0 as usize];
2972        if matches!(n.kind, Kind::Footnote | Kind::Endnote | Kind::Aside) && !n.deixis {
2973            return Some(node);
2974        }
2975        self.note_body_of(node)
2976    }
2977
2978    /// The nearest prose block enclosing `node` (itself included).
2979    fn block_of(&self, node: NodeId) -> Option<NodeId> {
2980        let mut cur = Some(node);
2981        while let Some(c) = cur {
2982            if self.is_prose_block(c.0 as usize) {
2983                return Some(c);
2984            }
2985            cur = self.nodes[c.0 as usize].parent;
2986        }
2987        None
2988    }
2989
2990    /// The note body a node stands in, if any: the nearest
2991    /// ancestor that is a footnote, endnote or aside body (not the
2992    /// callout). A node in a body cites as the body's callout.
2993    fn note_body_of(&self, node: NodeId) -> Option<NodeId> {
2994        let mut cur = self.nodes[node.0 as usize].parent;
2995        while let Some(p) = cur {
2996            let n = &self.nodes[p.0 as usize];
2997            if matches!(n.kind, Kind::Footnote | Kind::Endnote | Kind::Aside) && !n.deixis {
2998                return Some(p);
2999            }
3000            cur = n.parent;
3001        }
3002        None
3003    }
3004
3005    /// The document's prose as one text — every prose block's `::`,
3006    /// blank-line separated, in reading order — the text an
3007    /// external annotator reads and a CoNLL-U source aligns to.
3008    pub fn corpus_text(&self) -> String {
3009        self.prose_blocks()
3010            .iter()
3011            .map(|&b| self.nodes[b.0 as usize].prose.as_str())
3012            .collect::<Vec<_>>()
3013            .join("\n\n")
3014    }
3015
3016    /// Whether the corpus reading's tokens are already in place.
3017    pub fn is_tokenized(&self) -> bool {
3018        self.tiered || !self.ann.is_empty()
3019    }
3020
3021    /// [`tokenize`](Self::tokenize) with sentence bonds: a `.desm`
3022    /// set (the `syndesmos` crate) whose abbreviations and patterns
3023    /// undo UAX #29 breaks.
3024    pub fn tokenize_with(&mut self, bonds: Option<&syndesmos::Syndesmos>) {
3025        self.mutated();
3026        use unicode_segmentation::UnicodeSegmentation;
3027        self.token_index = Default::default();
3028        // Plain prose: nothing declared its tokens, so the tier is
3029        // virtual — computed per block when a query reaches it,
3030        // nothing stored now.
3031        let declared = self.prose_blocks().iter().any(|b| {
3032            let n = &self.nodes[b.0 as usize];
3033            !n.declared_tokens.is_empty() || !n.declared_sentences.is_empty()
3034        });
3035        if !declared && !self.is_tokenized() {
3036            self.tiered = true;
3037            self.bonds = bonds.cloned();
3038            self.tiers.borrow_mut().clear();
3039            self.attribute_spans();
3040            return;
3041        }
3042        let mut n: u32 = 0;
3043        let mut sentence: u32 = 0;
3044        for id in self.tier_blocks() {
3045            if self.heading_text(id).is_some() {
3046                self.push_heading_sentence(id, &mut n, &mut sentence);
3047                continue;
3048            }
3049            let block = NodeId(id as u64);
3050            let prose = self.nodes[id].prose.clone();
3051            // The sentence tier: one derived sentence
3052            // node per span, under the block; a block the
3053            // segmenter leaves whole is one sentence.
3054            // The parsing pack: the source's own
3055            // sentences and tokens stand in for the segmenter's
3056            // where the source declared them.
3057            let declared_sentences = std::mem::take(&mut self.nodes[id].declared_sentences);
3058            let mut declared_tokens = std::mem::take(&mut self.nodes[id].declared_tokens);
3059            declared_tokens.sort_by_key(|t| t.lo);
3060            let mut spans = if declared_sentences.is_empty() {
3061                sentence_spans(&prose, bonds)
3062            } else {
3063                declared_sentences
3064                    .iter()
3065                    .map(|(lo, hi, _)| (*lo as usize, *hi as usize))
3066                    .filter(|(lo, hi)| lo < hi && *hi <= prose.len())
3067                    .collect()
3068            };
3069            if spans.is_empty() {
3070                spans.push((0, prose.len()));
3071            }
3072            let sentence_ids: Vec<Option<String>> = if declared_sentences.is_empty() {
3073                Vec::new()
3074            } else {
3075                declared_sentences
3076                    .iter()
3077                    .map(|(_, _, sid)| Some(sid.clone()))
3078                    .collect()
3079            };
3080            let sentences: Vec<(usize, usize, u32, NodeId)> = spans
3081                .into_iter()
3082                .enumerate()
3083                .map(|(k, (lo, hi))| {
3084                    sentence += 1;
3085                    let node = self.push_derived_sentence(block, &prose[lo..hi], sentence);
3086                    if let Some(Some(sid)) = sentence_ids.get(k) {
3087                        self.nodes[node.0 as usize]
3088                            .fields
3089                            .push(("sent_id".to_string(), sid.clone()));
3090                    }
3091                    (lo, hi, sentence, node)
3092                })
3093                .collect();
3094            // Declared tokens by onym, for the heads.
3095            let mut by_onym: Vec<(String, NodeId)> = Vec::new();
3096            let mut heads: Vec<(NodeId, String)> = Vec::new();
3097            let mut next_declared = 0usize;
3098            let mut skip_until = 0usize;
3099            for (at, seg) in prose.split_word_bound_indices() {
3100                if at < skip_until {
3101                    continue;
3102                }
3103                // A declared token starting here takes the place
3104                // of the segmenter's, however many word bounds
3105                // it spans.
3106                while next_declared < declared_tokens.len()
3107                    && (declared_tokens[next_declared].lo as usize) < at
3108                {
3109                    next_declared += 1;
3110                }
3111                if let Some(d) = declared_tokens.get(next_declared)
3112                    && d.lo as usize == at
3113                    && (d.hi as usize) > at
3114                    && (d.hi as usize) <= prose.len()
3115                {
3116                    let d = d.clone();
3117                    next_declared += 1;
3118                    skip_until = d.hi as usize;
3119                    let form = &prose[d.lo as usize..d.hi as usize];
3120                    n += 1;
3121                    let (in_sentence, span, parent) = sentences
3122                        .iter()
3123                        .find(|(lo, hi, _, _)| *lo <= at && at < *hi)
3124                        .or_else(|| sentences.last())
3125                        .map(|(lo, hi, s, node)| (*s, (*lo as u32, *hi as u32), *node))
3126                        .expect("a block has at least one sentence");
3127                    let _ = span;
3128                    let tok = self.push_parsed(parent, form, &d, n, in_sentence, at as u32);
3129                    if let Some(o) = &d.onym {
3130                        by_onym.push((o.clone(), tok));
3131                    }
3132                    if let Some(h) = d.parsings.first().and_then(|p| p.kephale.clone()) {
3133                        heads.push((tok, h));
3134                    }
3135                    continue;
3136                }
3137                if seg.chars().all(char::is_whitespace) {
3138                    continue;
3139                }
3140                n += 1;
3141                let (in_sentence, _, parent) = sentences
3142                    .iter()
3143                    .find(|(lo, hi, _, _)| *lo <= at && at < *hi)
3144                    .or_else(|| sentences.last())
3145                    .map(|(lo, hi, s, node)| (*s, (*lo as u32, *hi as u32), *node))
3146                    .expect("a block has at least one sentence");
3147                self.push_plain_row(parent, seg, n, in_sentence, at as u32);
3148            }
3149            // The declared heads (`->head`, `<-head`), by onym.
3150            let linked: Vec<(NodeId, NodeId)> = heads
3151                .into_iter()
3152                .filter_map(|(tok, h)| {
3153                    by_onym
3154                        .iter()
3155                        .find(|(o, _)| *o == h)
3156                        .map(|(_, head)| (tok, *head))
3157                })
3158                .collect();
3159            self.link_declared_heads(&linked);
3160        }
3161        self.attribute_spans();
3162    }
3163
3164    /// The literary reading: give every span the tokens its byte
3165    /// range covers (`->token`), and every token its spans,
3166    /// outermost first — after the tokens exist.
3167    fn attribute_spans(&mut self) {
3168        self.mutated();
3169        if self.tiered {
3170            self.attribute_spans_virtual();
3171            return;
3172        }
3173        let spans: Vec<NodeId> = (0..self.nodes.len())
3174            .filter(|&i| self.nodes[i].kind == Kind::Span && self.nodes[i].at.is_some())
3175            .map(|i| NodeId(i as u64))
3176            .collect();
3177        if spans.is_empty() {
3178            return;
3179        }
3180        for span in spans {
3181            let (lo, hi) = self.nodes[span.0 as usize].at.expect("filtered");
3182            // The flow block the span belongs to: up past the
3183            // enclosing spans.
3184            let mut block = span;
3185            while self.nodes[block.0 as usize].kind == Kind::Span
3186                && let Some(p) = self.nodes[block.0 as usize].parent
3187            {
3188                block = p;
3189            }
3190            let covered: Vec<NodeId> = self
3191                .ann_under(block.0 as usize)
3192                .into_iter()
3193                .filter(|&r| {
3194                    let at = self.ann[r as usize].at;
3195                    lo <= at && at < hi
3196                })
3197                .map(ann::ann_id)
3198                .collect();
3199            self.nodes[span.0 as usize].span_tokens = covered;
3200        }
3201    }
3202
3203    /// One sentence of the corpus reading's sentence tier (ruling
3204    /// #64), under `block`: a `sentence` node the segmentation in
3205    /// force derived, its `::` the span's text, its ordinal the
3206    /// document-wide sentence count. Annotation, never a block:
3207    /// the block's own prose is unchanged and the renderers never
3208    /// see it.
3209    /// A heading's sentence on a stored tier (a CoNLL-U sidecar's, a
3210    /// parsing pack's): headings are sentences under every
3211    /// segmentation, so the title and each heading get one derived
3212    /// sentence under their node, before its content, its tokens the
3213    /// segmenter's as rows, counted in sequence with the rest.
3214    fn push_heading_sentence(&mut self, block: usize, n: &mut u32, sentence: &mut u32) {
3215        use unicode_segmentation::UnicodeSegmentation;
3216        let Some(text) = self.heading_text(block).map(str::to_string) else {
3217            return;
3218        };
3219        *sentence += 1;
3220        let node = self.push_derived_sentence(NodeId(block as u64), &text, *sentence);
3221        // Before the content: the node was appended last.
3222        let kids = &mut self.nodes[block].children;
3223        if let Some(pos) = kids.iter().position(|&c| c == node) {
3224            let id = kids.remove(pos);
3225            kids.insert(0, id);
3226        }
3227        for (at, seg) in text.split_word_bound_indices() {
3228            if seg.chars().all(char::is_whitespace) {
3229                continue;
3230            }
3231            *n += 1;
3232            self.push_plain_row(node, seg, *n, *sentence, at as u32);
3233        }
3234    }
3235
3236    fn push_derived_sentence(&mut self, block: NodeId, text: &str, ordinal: u32) -> NodeId {
3237        self.mutated();
3238        let node = push(&mut self.nodes, Kind::Sentence, block);
3239        let s = &mut self.nodes[node.0 as usize];
3240        s.text = text.trim().to_string();
3241        s.prose = s.text.clone();
3242        s.taxis = Some(ordinal as i64);
3243        s.derived = true;
3244        node
3245    }
3246
3247    /// The corpus reading from a CoNLL-U annotation (the corpus
3248    /// reading's second layer, carrying every column): the
3249    /// source's sentences and tokens replace the built-in
3250    /// segmentation wholesale, aligned to the document's prose by
3251    /// character offset — each token's form is found at the
3252    /// cursor, whitespace skipped — and every token carries the
3253    /// source's lemma, parts of speech, features, dependency
3254    /// relation, head edge, enhanced heads and miscellany. A form
3255    /// that does not align is an error naming the offset: the
3256    /// annotation must be of this text. A multiword range (`1-2
3257    /// don't`) aligns as the unit and its syntactic words share its
3258    /// span; an empty node (`8.1`) is not in the text and consumes
3259    /// none of it.
3260    pub fn annotate_conllu(&mut self, conllu: &str) -> Result<(), String> {
3261        self.mutated();
3262        if self.is_tokenized() {
3263            return Err("the document is already tokenized".into());
3264        }
3265        self.token_index = Default::default();
3266        let blocks = self.prose_blocks();
3267        // The document text and each block's range in it.
3268        let mut doc = String::new();
3269        let mut ranges: Vec<(usize, usize, NodeId)> = Vec::new();
3270        for (i, &b) in blocks.iter().enumerate() {
3271            if i > 0 {
3272                doc.push_str("\n\n");
3273            }
3274            let lo = doc.len();
3275            doc.push_str(&self.nodes[b.0 as usize].prose);
3276            ranges.push((lo, doc.len(), b));
3277        }
3278        if ranges.is_empty() {
3279            return Err("the document has no prose to annotate".into());
3280        }
3281        let parsed = parse_conllu(conllu)?;
3282        let mut cursor = 0usize;
3283        let mut block_at = 0usize;
3284        let mut n: u32 = 0;
3285        // The headings, emitted as sentences before the first block
3286        // that follows each; the sentence count runs through them.
3287        let mut headings: Vec<usize> = (0..self.nodes.len())
3288            .filter(|&i| self.heading_text(i).is_some())
3289            .collect();
3290        headings.reverse();
3291        let mut sentence_no: u32 = 0;
3292        let mut entered: Option<usize> = None;
3293        for sent in parsed.sentences.iter() {
3294            // The sentence's ordinal, taken when its first token is
3295            // placed — after the headings it comes after.
3296            let mut ordinal: u32 = 0;
3297            let mut made: Vec<(NodeId, &ConlluToken)> = Vec::new();
3298            // The (block, lo, hi) extents the sentence covers — and
3299            // the sentence node under each block: a
3300            // sentence the annotator runs across a block boundary
3301            // is one node per block, sharing the ordinal.
3302            let mut extents: Vec<(NodeId, usize, usize)> = Vec::new();
3303            let mut nodes_by_block: Vec<(NodeId, NodeId)> = Vec::new();
3304            // A multiword range in force: its span, last word id,
3305            // id and surface form.
3306            let mut range: Option<(usize, usize, u32, String, String)> = None;
3307            for tok in &sent.tokens {
3308                // A whitespace-only form (a tool's SPACE token) is
3309                // no token of the text.
3310                if !tok.empty && tok.form.trim().is_empty() {
3311                    continue;
3312                }
3313                let (lo, hi, mwt) = if tok.empty {
3314                    (cursor, cursor, None)
3315                } else if let Some((lo, hi, until, id, form)) = &range
3316                    && tok.range.is_none()
3317                    && tok.ord <= *until
3318                {
3319                    (*lo, *hi, Some((id.clone(), form.clone())))
3320                } else {
3321                    range = None;
3322                    while doc[cursor..].starts_with(char::is_whitespace) {
3323                        cursor += doc[cursor..].chars().next().map_or(0, char::len_utf8);
3324                    }
3325                    if !doc[cursor..].starts_with(tok.form.as_str()) {
3326                        let seen: String = doc[cursor..].chars().take(24).collect();
3327                        return Err(format!(
3328                            "CoNLL-U sentence {}, token {}: {:?} does not align with the text at offset {cursor} ({seen:?})",
3329                            sentence_no + 1,
3330                            tok.id,
3331                            tok.form
3332                        ));
3333                    }
3334                    let lo = cursor;
3335                    cursor += tok.form.len();
3336                    if let Some((_, b)) = tok.range {
3337                        range = Some((lo, cursor, b, tok.id.clone(), tok.form.clone()));
3338                        continue;
3339                    }
3340                    (lo, cursor, None)
3341                };
3342                if !tok.empty {
3343                    while block_at + 1 < ranges.len() && lo >= ranges[block_at].1 {
3344                        block_at += 1;
3345                    }
3346                }
3347                let (blo, _, block) = ranges[block_at];
3348                if entered != Some(block_at) {
3349                    // Entering a block: the headings before it first.
3350                    while let Some(&h) = headings.last()
3351                        && h < block.0 as usize
3352                    {
3353                        self.push_heading_sentence(h, &mut n, &mut sentence_no);
3354                        headings.pop();
3355                    }
3356                    entered = Some(block_at);
3357                }
3358                if ordinal == 0 {
3359                    sentence_no += 1;
3360                    ordinal = sentence_no;
3361                }
3362                if !tok.empty {
3363                    n += 1;
3364                }
3365                let sentence_node = match nodes_by_block.iter().find(|(b, _)| *b == block) {
3366                    Some((_, s)) => *s,
3367                    None => {
3368                        let s = self.push_derived_sentence(block, "", ordinal);
3369                        nodes_by_block.push((block, s));
3370                        s
3371                    }
3372                };
3373                let node = self.push_annotated(
3374                    sentence_node,
3375                    tok,
3376                    if tok.empty { 0 } else { n },
3377                    ordinal,
3378                    mwt,
3379                );
3380                if !tok.empty {
3381                    match extents.iter_mut().find(|(b, _, _)| *b == block) {
3382                        Some(e) => {
3383                            e.1 = e.1.min(lo - blo);
3384                            e.2 = e.2.max(hi - blo);
3385                        }
3386                        None => extents.push((block, lo - blo, hi - blo)),
3387                    }
3388                    if let Some(row) = ann::ann_row(node) {
3389                        self.ann[row as usize].at = (lo - blo) as u32;
3390                    }
3391                }
3392                made.push((node, tok));
3393            }
3394            // The sentence's span per block — on its node's text
3395            // and on each token — then the edges.
3396            for (block, s) in &nodes_by_block {
3397                if let Some((_, lo, hi)) = extents.iter().find(|(b, _, _)| b == block) {
3398                    let text = self.nodes[block.0 as usize].prose[*lo..*hi]
3399                        .trim()
3400                        .to_string();
3401                    let sn = &mut self.nodes[s.0 as usize];
3402                    sn.text = text.clone();
3403                    sn.prose = text;
3404                }
3405            }
3406            self.link_sentence(&made);
3407        }
3408        while let Some(h) = headings.pop() {
3409            self.push_heading_sentence(h, &mut n, &mut sentence_no);
3410        }
3411        let header = parsed
3412            .globals
3413            .iter()
3414            .find(|(k, _)| k == "Entity")
3415            .map(|(_, v)| v.as_str());
3416        self.decode_mentions(header);
3417        Ok(())
3418    }
3419
3420    /// A treebank read as a document: the file is the
3421    /// root; `# newdoc` opens a section, `# newpar` a paragraph,
3422    /// each sentence block a `sentence` node whose `::` is its
3423    /// `# text` (else the forms joined per `SpaceAfter`) and whose
3424    /// comments answer as properties. No tokens: the reading a
3425    /// bare `.conllu` or `text:` takes.
3426    pub fn parse_conllu_text(text: &str) -> Result<Self, String> {
3427        Self::from_conllu_set(&[(None, text)], false)
3428    }
3429
3430    /// A treebank read as a corpus: [`parse_conllu_text`]'s
3431    /// document with the file's own tokens as each sentence's
3432    /// children, in id order, every column carried — the reading
3433    /// `corpus:` takes on a `.conllu`.
3434    ///
3435    /// [`parse_conllu_text`]: Self::parse_conllu_text
3436    pub fn parse_conllu_corpus(text: &str) -> Result<Self, String> {
3437        Self::from_conllu_set(&[(None, text)], true)
3438    }
3439
3440    /// A set of treebank files read as one document — a UD
3441    /// treebank's train / dev / test files, a release directory:
3442    /// each file a level-1 `section` under the root whose
3443    /// `::lemma` is the name given, the file's own `# newdoc`
3444    /// sections nested beneath it. No tokens, as
3445    /// [`parse_conllu_text`](Self::parse_conllu_text).
3446    pub fn parse_conllu_text_set(files: &[(&str, &str)]) -> Result<Self, String> {
3447        let named: Vec<(Option<&str>, &str)> = files.iter().map(|(n, t)| (Some(*n), *t)).collect();
3448        Self::from_conllu_set(&named, false)
3449    }
3450
3451    /// [`parse_conllu_text_set`](Self::parse_conllu_text_set) with
3452    /// every file's tokens under its sentences: token positions,
3453    /// sentence ordinals and mention ordinals run across the whole
3454    /// set, as they do across one file.
3455    pub fn parse_conllu_corpus_set(files: &[(&str, &str)]) -> Result<Self, String> {
3456        let named: Vec<(Option<&str>, &str)> = files.iter().map(|(n, t)| (Some(*n), *t)).collect();
3457        Self::from_conllu_set(&named, true)
3458    }
3459
3460    fn from_conllu_set(files: &[(Option<&str>, &str)], tokens: bool) -> Result<Self, String> {
3461        let mut parsed: Vec<(Option<&str>, ConlluDoc)> = Vec::with_capacity(files.len());
3462        for (name, text) in files {
3463            let doc = parse_conllu(text).map_err(|e| match name {
3464                Some(n) => format!("{n}: {e}"),
3465                None => e,
3466            })?;
3467            parsed.push((*name, doc));
3468        }
3469        let mut nodes = vec![Node::new(Kind::Document, None)];
3470        let root = NodeId(0);
3471        // Every sentence node, with the file and sentence it came from.
3472        let mut sentences: Vec<(NodeId, usize, usize)> = Vec::new();
3473        for (fi, (name, doc)) in parsed.iter().enumerate() {
3474            let (base, doc_level) = match name {
3475                Some(n) => {
3476                    let sec = push(&mut nodes, Kind::Section, root);
3477                    nodes[sec.0 as usize].lemma = Some(n.to_string());
3478                    nodes[sec.0 as usize].level = Some(1);
3479                    (sec, 2)
3480                }
3481                None => (root, 1),
3482            };
3483            let mut section: Option<NodeId> = None;
3484            let mut para: Option<NodeId> = None;
3485            for (si, s) in doc.sentences.iter().enumerate() {
3486                if let Some(id) = &s.newdoc {
3487                    let sec = push(&mut nodes, Kind::Section, base);
3488                    nodes[sec.0 as usize].lemma = id.clone();
3489                    nodes[sec.0 as usize].level = Some(doc_level);
3490                    section = Some(sec);
3491                    para = None;
3492                }
3493                if s.newpar.is_some() {
3494                    let p = push(&mut nodes, Kind::Paragraph, section.unwrap_or(base));
3495                    para = Some(p);
3496                }
3497                let parent = para.or(section).unwrap_or(base);
3498                let sent = push(&mut nodes, Kind::Sentence, parent);
3499                nodes[sent.0 as usize].taxis = Some(sentences.len() as i64 + 1);
3500                let text = s
3501                    .comments
3502                    .iter()
3503                    .find(|(k, _)| k == "text")
3504                    .map(|(_, v)| v.clone())
3505                    .unwrap_or_else(|| conllu_surface(&s.tokens));
3506                nodes[sent.0 as usize].text = text;
3507                nodes[sent.0 as usize].fields = s.comments.clone();
3508                sentences.push((sent, fi, si));
3509            }
3510        }
3511        flatten_prose(&mut nodes);
3512        let mut model = TextModel {
3513            nodes,
3514            root,
3515            caches: Default::default(),
3516            onyms: Default::default(),
3517            document_url: None,
3518            link_base: None,
3519            document_path: None,
3520            cite_scheme: None,
3521            notes_omitted: false,
3522            orthography: None,
3523            bib_aliases: Default::default(),
3524            head: HeadMeta::default(),
3525            token_index: Default::default(),
3526            mentions: Vec::new(),
3527            tiered: false,
3528            bonds: None,
3529            tiers: Default::default(),
3530            given: Default::default(),
3531            ords: Default::default(),
3532            virt_fields: Default::default(),
3533            virt_character: Default::default(),
3534            virt_place: Default::default(),
3535            ann: Vec::new(),
3536            ann_names: Default::default(),
3537            ann_of: Default::default(),
3538            ann_deps: Default::default(),
3539            ann_edeps: Default::default(),
3540        };
3541        if tokens {
3542            let mut n: u32 = 0;
3543            for (i, (sent, fi, si)) in sentences.into_iter().enumerate() {
3544                let s = &parsed[fi].1.sentences[si];
3545                let ordinal = i as u32 + 1;
3546                let len = model.nodes[sent.0 as usize].prose.len() as u32;
3547                let mut made: Vec<(NodeId, &ConlluToken)> = Vec::new();
3548                let mut cursor = 0usize;
3549                // A multiword range in force: last word id, id, form.
3550                let mut range: Option<(u32, String, String)> = None;
3551                for tok in &s.tokens {
3552                    if let Some((_, b)) = tok.range {
3553                        range = Some((b, tok.id.clone(), tok.form.clone()));
3554                        continue;
3555                    }
3556                    let mwt = match &range {
3557                        Some((until, id, form)) if !tok.empty && tok.ord <= *until => {
3558                            Some((id.clone(), form.clone()))
3559                        }
3560                        _ => {
3561                            range = None;
3562                            None
3563                        }
3564                    };
3565                    if !tok.empty {
3566                        n += 1;
3567                    }
3568                    let in_range = mwt.is_some();
3569                    let node = model.push_annotated(
3570                        sent,
3571                        tok,
3572                        if tok.empty { 0 } else { n },
3573                        ordinal,
3574                        mwt,
3575                    );
3576                    let _ = len;
3577                    // The token's offset along the sentence's surface
3578                    // (`# text`, or the forms joined by SpaceAfter):
3579                    // the next occurrence of the form from the cursor.
3580                    if !tok.empty
3581                        && let Some(row) = ann::ann_row(node)
3582                    {
3583                        let surface = &model.nodes[sent.0 as usize].prose;
3584                        if let Some(found) =
3585                            surface[cursor.min(surface.len())..].find(tok.form.as_str())
3586                        {
3587                            let at = cursor + found;
3588                            model.ann[row as usize].at = at as u32;
3589                            if !in_range {
3590                                cursor = at + tok.form.len();
3591                            }
3592                        }
3593                    }
3594                    made.push((node, tok));
3595                }
3596                model.link_sentence(&made);
3597            }
3598            // One `# global.Entity` header serves the set: a
3599            // release declares it identically in every file.
3600            let header = parsed
3601                .iter()
3602                .flat_map(|(_, d)| d.globals.iter())
3603                .find(|(k, _)| k == "Entity")
3604                .map(|(_, v)| v.as_str());
3605            model.decode_mentions(header);
3606        }
3607        Ok(model)
3608    }
3609}
3610
3611/// The innermost span under `flow` whose byte range contains
3612/// `lo..hi` — the parent a nested span (a name inside a speech)
3613/// lands under. Spans arrive in pre-order, so the deepest match
3614/// is the right one.
3615fn enclosing_span(nodes: &[Node], flow: NodeId, lo: u32, hi: u32) -> Option<NodeId> {
3616    let mut best: Option<NodeId> = None;
3617    let mut at = flow;
3618    loop {
3619        let next = nodes[at.0 as usize].children.iter().rev().find_map(|&c| {
3620            let n = &nodes[c.0 as usize];
3621            match (n.kind, n.at) {
3622                (Kind::Span, Some((slo, shi)))
3623                    if slo <= lo && hi <= shi && (slo, shi) != (lo, hi) =>
3624                {
3625                    Some(c)
3626                }
3627                _ => None,
3628            }
3629        });
3630        match next {
3631            Some(c) => {
3632                best = Some(c);
3633                at = c;
3634            }
3635            None => return best,
3636        }
3637    }
3638}
3639
3640/// Every value under `name` among a node's fields: one value, a
3641/// list where the key repeats, nothing where it is absent.
3642fn field_values(fields: &[(String, String)], name: &str) -> Option<Value> {
3643    let vs: Vec<Value> = fields
3644        .iter()
3645        .filter(|(k, _)| k == name)
3646        .map(|(_, v)| Value::Str(v.clone()))
3647        .collect();
3648    match vs.len() {
3649        0 => None,
3650        1 => vs.into_iter().next(),
3651        _ => Some(Value::list(vs)),
3652    }
3653}
3654
3655fn push(nodes: &mut Vec<Node>, kind: Kind, parent: NodeId) -> NodeId {
3656    let id = NodeId(nodes.len() as u64);
3657    nodes.push(Node::new(kind, Some(parent)));
3658    nodes[parent.0 as usize].children.push(id);
3659    id
3660}
3661
3662/// Denormalize a table into nested lists (see the module doc).
3663/// The column name lands as the cell's `::lemma` — a cell's own
3664/// label (a row's `th`) wins over the positional header entry —
3665/// and never as folded text: addressing is property projection,
3666/// the flattening rule alone spells `lemma: value`.
3667fn lower_table(
3668    nodes: &mut Vec<Node>,
3669    parent: NodeId,
3670    lemma: Option<String>,
3671    headers: Option<Vec<String>>,
3672    rows: Vec<Vec<Cell>>,
3673) {
3674    let list = push(nodes, Kind::OrderedList, parent);
3675    {
3676        let n = &mut nodes[list.0 as usize];
3677        n.table = true;
3678        n.lemma = lemma.map(|l| normalize_ws(&l)).filter(|l| !l.is_empty());
3679    }
3680    for (i, row) in rows.into_iter().enumerate() {
3681        let item = push(nodes, Kind::OrderedItem, list);
3682        nodes[item.0 as usize].taxis = Some(i as i64 + 1);
3683        nodes[item.0 as usize].row = true;
3684        let cells = push(nodes, Kind::UnorderedList, item);
3685        for (j, cell) in row.into_iter().enumerate() {
3686            let value = normalize_ws(&cell.text);
3687            if value.is_empty() {
3688                continue;
3689            }
3690            let label = cell
3691                .label
3692                .as_deref()
3693                .or_else(|| headers.as_ref().and_then(|h| h.get(j)).map(|h| h.as_str()))
3694                .map(normalize_ws)
3695                .filter(|h| !h.is_empty());
3696            let cell_item = push(nodes, Kind::UnorderedItem, cells);
3697            let n = &mut nodes[cell_item.0 as usize];
3698            n.cell = true;
3699            n.lemma = label;
3700            n.text = value;
3701        }
3702    }
3703}
3704
3705/// Lower a verse block: `verse` → `strophe`s →
3706/// `stichos` lines. The stichos taxis numbers lines 1-based,
3707/// continuously across strophes — Iliad 1.34 is a taxis; the
3708/// strophe taxis is its ordinal. Empty lines are dropped (the
3709/// strophe boundary carries the separation).
3710fn lower_verse(
3711    nodes: &mut Vec<Node>,
3712    parent: NodeId,
3713    lemma: Option<String>,
3714    strophes: Vec<Vec<String>>,
3715    hypograph: Option<String>,
3716) -> NodeId {
3717    let verse = push(nodes, Kind::Verse, parent);
3718    {
3719        let n = &mut nodes[verse.0 as usize];
3720        n.lemma = lemma.map(|l| normalize_ws(&l)).filter(|l| !l.is_empty());
3721        n.hypograph = hypograph
3722            .map(|h| normalize_ws(&h))
3723            .filter(|h| !h.is_empty());
3724    }
3725    let mut line_no = 0i64;
3726    for (i, strophe) in strophes.into_iter().enumerate() {
3727        let sid = push(nodes, Kind::Strophe, verse);
3728        nodes[sid.0 as usize].taxis = Some(i as i64 + 1);
3729        for line in strophe {
3730            let line = line.trim_end().to_string();
3731            if line.trim().is_empty() {
3732                continue;
3733            }
3734            line_no += 1;
3735            let lid = push(nodes, Kind::Stichos, sid);
3736            let n = &mut nodes[lid.0 as usize];
3737            n.taxis = Some(line_no);
3738            n.text = line;
3739        }
3740    }
3741    verse
3742}
3743
3744/// Compute every node's flattened prose: lemma first, then the
3745/// node's own text, then its children's prose in order, then the
3746/// hypograph, block-joined with newlines. On a list *item*, the
3747/// lemma joins the rest with `: ` instead — an item's lemma names
3748/// its content inline (a table cell reads `Outcome: Emus won`, a
3749/// definition reads `term: description`), where a section's lemma
3750/// opens its block. Children always carry larger indices than
3751/// their parents (nodes are interned in document order), so one
3752/// reverse index scan suffices — no recursion.
3753fn flatten_prose(nodes: &mut [Node]) {
3754    for i in (0..nodes.len()).rev() {
3755        let inline_lemma = matches!(nodes[i].kind, Kind::UnorderedItem | Kind::OrderedItem);
3756        let mut lemma_part: Option<String> = None;
3757        let mut parts: Vec<String> = Vec::new();
3758        if let Some(lemma) = &nodes[i].lemma
3759            && !lemma.is_empty()
3760        {
3761            if inline_lemma {
3762                lemma_part = Some(lemma.clone());
3763            } else if matches!(nodes[i].kind, Kind::Paragraph | Kind::Dialogue) {
3764                // A speech's lemma is its speaker — who said it,
3765                // not what was said: `::lemma`, never prose, so
3766                // the words alone are read, tokenized, counted.
3767            } else {
3768                parts.push(lemma.clone());
3769            }
3770        }
3771        if !nodes[i].text.is_empty() {
3772            parts.push(nodes[i].text.clone());
3773        }
3774        let mut child_parts: Vec<String> = Vec::new();
3775        for &child in nodes[i].children.clone().iter() {
3776            // A ref's recorded link text is already part of the
3777            // flow it was met in; an anchor is invisible. Neither
3778            // adds prose (the deixis/index-mark rule).
3779            if matches!(
3780                nodes[child.0 as usize].kind,
3781                Kind::Ref | Kind::Cit | Kind::Anchor | Kind::Span | Kind::Milestone
3782            ) {
3783                continue;
3784            }
3785            let prose = &nodes[child.0 as usize].prose;
3786            if !prose.is_empty() {
3787                child_parts.push(prose.clone());
3788            }
3789        }
3790        // Ruling #37: a verse block separates its strophes by a
3791        // blank line (lemma and hypograph still join normally) —
3792        // byte-compatible with the old verbatim lowering.
3793        if nodes[i].kind == Kind::Verse {
3794            let joined = child_parts.join("\n\n");
3795            if !joined.is_empty() {
3796                parts.push(joined);
3797            }
3798        } else if nodes[i].kind == Kind::Paragraph
3799            && !child_parts.is_empty()
3800            && nodes[i]
3801                .children
3802                .iter()
3803                .all(|c| nodes[c.0 as usize].kind == Kind::Sentence)
3804        {
3805            // Ruling #63: a treebank paragraph is its sentences
3806            // run together, as the prose would read.
3807            parts.push(child_parts.join(" "));
3808        } else {
3809            parts.extend(child_parts);
3810        }
3811        if let Some(hypograph) = &nodes[i].hypograph
3812            && !hypograph.is_empty()
3813        {
3814            parts.push(hypograph.clone());
3815        }
3816        let mut prose = parts.join("\n");
3817        if let Some(lemma) = lemma_part {
3818            prose = if prose.is_empty() {
3819                lemma
3820            } else {
3821                format!("{lemma}: {prose}")
3822            };
3823        }
3824        nodes[i].prose = prose;
3825    }
3826}
3827
3828impl TextModel {
3829    /// The body prose: everything between the lemma and the
3830    /// hypograph — the simmere anatomy's third member, derived
3831    /// from the flattened prose by construction (the lemma joins
3832    /// a block on its own line, an item with `: `; the hypograph
3833    /// closes on its own line).
3834    fn grammata(&self, node: NodeId) -> String {
3835        let n = &self.nodes[node.0 as usize];
3836        let mut s = n.prose.as_str();
3837        if let Some(lemma) = &n.lemma
3838            && !lemma.is_empty()
3839            && let Some(rest) = s.strip_prefix(lemma.as_str())
3840        {
3841            s = rest
3842                .strip_prefix(": ")
3843                .or_else(|| rest.strip_prefix('\n'))
3844                .unwrap_or(rest);
3845        }
3846        if let Some(h) = &n.hypograph
3847            && !h.is_empty()
3848            && let Some(rest) = s.strip_suffix(h.as_str())
3849        {
3850            s = rest.strip_suffix('\n').unwrap_or(rest);
3851        }
3852        s.trim_end().to_string()
3853    }
3854}
3855
3856impl TextModel {
3857    /// The dictionary vocabulary: the abbreviations list's item
3858    /// whose lemma is `text` — what a usage label resolves to.
3859    fn abbreviation(&self, text: &str) -> Option<NodeId> {
3860        let text = text.trim();
3861        if text.is_empty() {
3862            return None;
3863        }
3864        let list = self.nodes.iter().position(|n| {
3865            matches!(n.kind, Kind::UnorderedList | Kind::OrderedList)
3866                && n.fields
3867                    .iter()
3868                    .any(|(k, v)| k == "kind" && v == "abbreviations")
3869        })?;
3870        // An item's lemma may hold several spellings, comma-separated
3871        // (`adj., a.`): any of them is the label's.
3872        self.nodes[list].children.iter().copied().find(|&c| {
3873            self.nodes[c.0 as usize]
3874                .lemma
3875                .as_deref()
3876                .is_some_and(|l| l.split(',').any(|s| s.trim() == text))
3877        })
3878    }
3879}
3880
3881impl AstAdapter for TextModel {
3882    /// The shelf knows how many documents, sections and tokens a
3883    /// step reaches; nothing is read again.
3884    fn estimate(&self, scope: &[&str], plan: &quarb::StepPlan<'_>) -> Option<quarb::Estimate> {
3885        quarb::adapter::arena_estimate(self, scope, plan, quarb::Latency::Memory, None)
3886    }
3887
3888    fn root(&self) -> NodeId {
3889        self.root
3890    }
3891
3892    /// In memory: an unscoped reverse resolution or value match may
3893    /// walk the arbor.
3894    fn walks_for_references(&self) -> bool {
3895        true
3896    }
3897
3898    fn children(&self, node: NodeId) -> Vec<NodeId> {
3899        if let Some(v) = tier::virt(node) {
3900            return self.v_children(v);
3901        }
3902        let mut c = self.nodes[node.0 as usize].children.clone();
3903        if self.virtual_block(node.0 as usize) {
3904            let sentences = self.tier_sentences(node.0 as usize);
3905            if self.heading_text(node.0 as usize).is_some() {
3906                // A heading's sentence stands before the content.
3907                c.splice(0..0, sentences);
3908            } else {
3909                c.extend(sentences);
3910            }
3911        }
3912        if let Some(rows) = self.ann_of.get(&(node.0 as usize)) {
3913            c.extend(rows.iter().map(|&r| ann::ann_id(r)));
3914        }
3915        c
3916    }
3917
3918    fn name(&self, node: NodeId) -> Option<String> {
3919        if let Some(v) = tier::virt(node) {
3920            return Self::v_name(v);
3921        }
3922        let n = &self.nodes[node.0 as usize];
3923        if n.kind == Kind::Span {
3924            return n.span_kind.clone();
3925        }
3926        n.kind.name().map(str::to_string)
3927    }
3928
3929    fn parent(&self, node: NodeId) -> Option<NodeId> {
3930        if let Some(v) = tier::virt(node) {
3931            return self.v_parent(v);
3932        }
3933        self.nodes[node.0 as usize].parent
3934    }
3935
3936    /// The positional index: `//token[::lower
3937    /// = "word"]` below any node answers from the form → tokens
3938    /// map, scoped to the node's subtree. Only a plain word is
3939    /// answered — a literal with a numeric, temporal, durational or
3940    /// unital reading is left to the walk, whose equality reads it.
3941    fn descendants_where(
3942        &self,
3943        node: NodeId,
3944        name: &str,
3945        property: &str,
3946        value: &Value,
3947    ) -> Option<Vec<(NodeId, usize)>> {
3948        if name != "token" || property != "lower" {
3949            return None;
3950        }
3951        let Value::Str(word) = value else {
3952            return None;
3953        };
3954        if word.is_empty() || !word.chars().all(|c| c.is_alphabetic() || "’'-".contains(c)) {
3955            return None;
3956        }
3957        if tier::virt(node).is_some() {
3958            return None;
3959        }
3960        let index = self.token_index.get_or_init(|| {
3961            if self.tiered {
3962                return self.tier_index();
3963            }
3964            let mut map: std::collections::HashMap<String, Vec<NodeId>> =
3965                std::collections::HashMap::new();
3966            for (i, r) in self.ann.iter().enumerate() {
3967                map.entry(self.lower(self.ann_names.get(r.form)))
3968                    .or_default()
3969                    .push(ann::ann_id(i as u32));
3970            }
3971            map
3972        });
3973        let Some(hits) = index.get(word.as_str()) else {
3974            return Some(Vec::new());
3975        };
3976        let mut out = Vec::new();
3977        for &t in hits {
3978            // The depth below `node`, if `t` is in its subtree.
3979            let mut depth = 0usize;
3980            let mut cur = t;
3981            while let Some(p) = self.any_parent(cur) {
3982                depth += 1;
3983                if p == node {
3984                    out.push((t, depth));
3985                    break;
3986                }
3987                cur = p;
3988            }
3989        }
3990        Some(out)
3991    }
3992
3993    /// The `<block>` family on every block node, plus `<table>` on
3994    /// a list that denormalizes a table. Kinds are node names, not
3995    /// traits.
3996    fn traits(&self, node: NodeId) -> Vec<String> {
3997        if let Some(v) = tier::virt(node) {
3998            return self.v_traits(v);
3999        }
4000        let n = &self.nodes[node.0 as usize];
4001        let mut out = Vec::new();
4002        // The document wears its declared classification: one
4003        // trait per tag and one for the category — the LDAP rule,
4004        // where a multi-valued classifying attribute is the node's
4005        // traits — each in its own namespace (`<tag:pandas>`,
4006        // `<category:Guides>`), so a tag can never collide with a
4007        // structural trait like `<table>` or `<note>`, and spelled
4008        // as declared, case kept.
4009        if n.kind == Kind::Document {
4010            for (ns, t) in self
4011                .head
4012                .tags
4013                .iter()
4014                .map(|t| ("tag", t))
4015                .chain(self.head.category.iter().map(|c| ("category", c)))
4016            {
4017                let name = HeadMeta::trait_name(t);
4018                if name.is_empty() {
4019                    continue;
4020                }
4021                let name = format!("{ns}:{name}");
4022                if !out.contains(&name) {
4023                    out.push(name);
4024                }
4025            }
4026        }
4027        // A callout is inline apparatus, not a block;
4028        // so is an index mark. Strophes and stichos
4029        // lines are sub-block structure (litogramma's
4030        // own family rule) — the verse block carries the trait.
4031        if n.kind != Kind::Document
4032            && !n.deixis
4033            && !n.derived
4034            && !matches!(
4035                n.kind,
4036                Kind::IndexMark
4037                    | Kind::Ref
4038                    | Kind::Cit
4039                    | Kind::Anchor
4040                    | Kind::Strophe
4041                    | Kind::Stichos
4042                    | Kind::Span
4043                    | Kind::Milestone
4044                    | Kind::Character
4045                    | Kind::Place
4046            )
4047        {
4048            out.push("block".to_string());
4049        }
4050        // The literary reading: a simmere's genoses are its
4051        // traits (`//quotation<said>`, `//annotation<persname>`).
4052        for g in &n.genoses {
4053            if !out.contains(g) {
4054                out.push(g.clone());
4055            }
4056        }
4057        // The corpus reading's sentence tier: a
4058        // sentence the segmentation derived, as against a
4059        // treebank's declared one.
4060        if n.derived {
4061            out.push("derived".to_string());
4062        }
4063        // The reference vocabulary (atrep's semantic traits): a
4064        // dangling mention keeps its node and the linter finds it;
4065        // `<target>` marks what bears a referable name — anchors
4066        // and labeled blocks, never the apparatus families (their
4067        // onyms pair within the family, not in this namespace).
4068        if matches!(n.kind, Kind::Ref | Kind::Cit) && n.dangling {
4069            out.push("dangling".to_string());
4070        }
4071        // A structured entry's genus (liber, commentarius, …)
4072        // surfaces as its trait, the genos rule.
4073        if n.kind == Kind::Bib
4074            && let Some(g) = &n.genus
4075        {
4076            out.push(g.clone());
4077        }
4078        if n.kind == Kind::Anchor
4079            || (n.onym.is_some()
4080                && !n.deixis
4081                && !matches!(
4082                    n.kind,
4083                    Kind::Footnote
4084                        | Kind::Endnote
4085                        | Kind::Aside
4086                        | Kind::IndexMark
4087                        | Kind::Ref
4088                        | Kind::Cit
4089                        | Kind::Bib
4090                        | Kind::Span
4091                        | Kind::Milestone
4092                ))
4093        {
4094            out.push("target".to_string());
4095        }
4096        if n.kind == Kind::Cit && n.locator.is_some() {
4097            out.push("locator".to_string());
4098        }
4099        if matches!(n.kind, Kind::Footnote | Kind::Endnote | Kind::Aside) {
4100            // The body IS the note — `<note>` marks it, whichever
4101            // note family; the callout carries `<deixis>` alone.
4102            // An aside body is content, not apparatus: no
4103            // `<note>`. No both-ends trait: `//footnote` already
4104            // gathers a family whole, `//*<deixis>` the callouts
4105            // across families, `//*<note>` the note bodies.
4106            if n.deixis {
4107                out.push("deixis".to_string());
4108            } else if n.kind != Kind::Aside {
4109                out.push("note".to_string());
4110            }
4111            if n.dangling {
4112                out.push("dangling".to_string());
4113            }
4114        }
4115        if n.table {
4116            out.push("table".to_string());
4117        }
4118        if n.row {
4119            out.push("row".to_string());
4120        }
4121        if n.cell {
4122            out.push("cell".to_string());
4123        }
4124        out
4125    }
4126
4127    /// `::lemma` (title), `::hypograph` (footer or attribution),
4128    /// `::taxis` (ordinal), `::text` (the flattened prose, same as
4129    /// the bare projection).
4130    /// The Greek anatomy — `::lemma`, `::grammata`, `::hypograph`,
4131    /// `::taxis` — plus the friendly aliases (`::title`, `::body`,
4132    /// `::attribution`, `::ord`), answered here because this
4133    /// adapter's property surface IS the vocabulary; on data
4134    /// adapters those spellings stay ordinary field names. The
4135    /// Greek is canon in docs and reflection preserves whichever
4136    /// spelling was written.
4137    fn property(&self, node: NodeId, name: &str) -> Option<Value> {
4138        if let Some(v) = tier::virt(node) {
4139            return self.v_property(v, name);
4140        }
4141        let n = &self.nodes[node.0 as usize];
4142        // The citation in force at any node.
4143        if name == "cite" {
4144            return self.cite(node).map(Value::Str);
4145        }
4146        // The document node answers its declared identity: the
4147        // curated core of what the head says.
4148        if n.kind == Kind::Document {
4149            let h = &self.head;
4150            match name {
4151                "title" | "lemma" => return h.title.clone().map(Value::Str),
4152                "description" => return h.description.clone().map(Value::Str),
4153                "author" => return h.author.clone().map(Value::Str),
4154                "category" => return h.category.clone().map(Value::Str),
4155                "tags" => {
4156                    return (!h.tags.is_empty())
4157                        .then(|| Value::list(h.tags.iter().cloned().map(Value::Str).collect()));
4158                }
4159                "published" => return h.published.clone(),
4160                "modified" => return h.modified.clone(),
4161                _ => {}
4162            }
4163        }
4164        // A structured entry answers its campi first — under the
4165        // canonical Latin name or any census alias (BibLaTeX's
4166        // field names included), case-insensitively — ahead of
4167        // the general vocabulary, so `::title` on a bib means
4168        // titulus, not the lemma alias.
4169        if n.kind == Kind::Bib && !n.fields.is_empty() {
4170            let canon = self
4171                .bib_aliases
4172                .get(&name.to_lowercase())
4173                .map(String::as_str)
4174                .unwrap_or(name);
4175            if let Some((_, v)) = n.fields.iter().find(|(k, _)| k == canon) {
4176                return Some(Value::Str(v.clone()));
4177            }
4178        }
4179        match name {
4180            // A link into the node, as far as the served form can
4181            // narrow it; the anchor it lands on; and the record a
4182            // client renders as a clickable result.
4183            "href" => return self.href(node).map(Value::Str),
4184            "anchor" => return self.nearest_onym(node).map(|(a, _)| Value::Str(a)),
4185            "link" => {
4186                let href = self.href(node)?;
4187                let text = match self.default_value(node) {
4188                    Some(Value::Str(t)) => t,
4189                    _ => String::new(),
4190                };
4191                return Some(Value::Record(vec![
4192                    (
4193                        "title".to_string(),
4194                        self.document_title().map(Value::Str).unwrap_or(Value::Null),
4195                    ),
4196                    ("href".to_string(), Value::Str(href)),
4197                    ("text".to_string(), Value::Str(text)),
4198                ]));
4199            }
4200            _ => {}
4201        }
4202        match name {
4203            // The corpus reading's token properties; the linguistic
4204            // ones answer only under a CoNLL-U annotation.
4205            // A treebank sentence: `::id` its sent_id, every other
4206            // comment under its key.
4207            "id" if n.kind == Kind::Sentence => n
4208                .fields
4209                .iter()
4210                .find(|(k, _)| k == "sent_id")
4211                .map(|(_, v)| Value::Str(v.clone())),
4212            "lemma" | "title" => n.lemma.clone().map(Value::Str),
4213            "onym" => n.onym.clone().map(Value::Str),
4214            // On an index mark the onym IS the term.
4215            "term" if n.kind == Kind::IndexMark => n.onym.clone().map(Value::Str),
4216            // A ref's word: the identifier as written. Never the
4217            // resolved content — that is what `-->` is for.
4218            "target" if matches!(n.kind, Kind::Ref | Kind::Cit) => n.target.clone().map(Value::Str),
4219            // A cit's pinpoint, as written: `\cite[I.1]{key}`,
4220            // `[@key, I.1]`.
4221            "locator" if n.kind == Kind::Cit => n.locator.clone().map(Value::Str),
4222            // A structured entry answers its campi (auctor,
4223            // titulus, annus, …) — the bibliogramma vocabulary,
4224            // canonicalized upstream.
4225            _ if n.kind == Kind::Bib => {
4226                let canon = self
4227                    .bib_aliases
4228                    .get(&name.to_lowercase())
4229                    .map(String::as_str)
4230                    .unwrap_or(name);
4231                n.fields
4232                    .iter()
4233                    .find(|(k, _)| k == canon)
4234                    .map(|(_, v)| Value::Str(v.clone()))
4235            }
4236            "hypograph" | "attribution" => n.hypograph.clone().map(Value::Str),
4237            "taxis" | "ord" => n.taxis.map(Value::Int),
4238            "grammata" | "body" => {
4239                let g = self.grammata(node);
4240                if g.is_empty() {
4241                    None
4242                } else {
4243                    Some(Value::Str(g))
4244                }
4245            }
4246            "text" => Some(Value::Str(n.prose.clone())),
4247            // Every `Key=Value` of FEATS, MISC and a CoNLL-U Plus
4248            // column, as the annotation spells it or case-folded.
4249            // A cast table's own annotation on the token (its
4250            // `prosopon`) answers when the annotation has no such
4251            // key.
4252            // A treebank sentence's `# key = value` comments, the
4253            // same two spellings.
4254            _ if n.kind == Kind::Sentence => n
4255                .fields
4256                .iter()
4257                .find(|(k, _)| k == name)
4258                .or_else(|| n.fields.iter().find(|(k, _)| k.eq_ignore_ascii_case(name)))
4259                .map(|(_, v)| Value::Str(v.clone())),
4260            // The literary reading: a span's, a speech's, a
4261            // paragraph's unseen metadata under the sim name of the
4262            // monosim that carried it (`::prosopon`), a list where
4263            // it repeats (`eidos`); a milestone's scheme and value.
4264            // (A list's own kind — the dictionary vocabulary's
4265            // abbreviations list — answers the same way.)
4266            _ if matches!(
4267                n.kind,
4268                Kind::Span
4269                    | Kind::Speech
4270                    | Kind::Dialogue
4271                    | Kind::Paragraph
4272                    | Kind::Milestone
4273                    | Kind::Character
4274                    | Kind::Place
4275                    | Kind::UnorderedList
4276                    | Kind::OrderedList
4277            ) && !n.fields.is_empty() =>
4278            {
4279                field_values(&n.fields, name)
4280            }
4281            _ => None,
4282        }
4283    }
4284
4285    /// The default projection is the flattened prose of the
4286    /// subtree — lemma first, hypograph last.
4287    fn default_value(&self, node: NodeId) -> Option<Value> {
4288        if let Some(v) = tier::virt(node) {
4289            return self.v_default_value(v);
4290        }
4291        let n = &self.nodes[node.0 as usize];
4292        if n.kind == Kind::IndexMark {
4293            // Invisible in the surrounding prose; its own
4294            // projection is the term it declares.
4295            return Some(Value::Str(n.onym.clone().unwrap_or_default()));
4296        }
4297        if n.deixis {
4298            // The atrep rule: a resolved reference projects as its
4299            // target's rendered form, degrading to the raw onym
4300            // when dangling.
4301            return Some(Value::Str(match n.note_edge {
4302                Some(body) => self.nodes[body.0 as usize].prose.clone(),
4303                None => n.onym.clone().unwrap_or_default(),
4304            }));
4305        }
4306        if n.kind == Kind::Ref {
4307            // The authored link text; LaTeX \ref has none, so the
4308            // atrep projection rule applies — the target's
4309            // rendered form (its lemma, else its prose), degrading
4310            // to the raw target when dangling.
4311            if !n.text.is_empty() {
4312                return Some(Value::Str(n.text.clone()));
4313            }
4314            return Some(Value::Str(match n.ref_edge {
4315                Some(t) => {
4316                    let b = &self.nodes[t.0 as usize];
4317                    b.lemma.clone().unwrap_or_else(|| b.prose.clone())
4318                }
4319                None => n.target.clone().unwrap_or_default(),
4320            }));
4321        }
4322        if n.kind == Kind::Bib && n.prose.is_empty() && !n.fields.is_empty() {
4323            // A structured entry's plain form: its field values in
4324            // source order — the full data, no citation styling.
4325            return Some(Value::Str(
4326                n.fields
4327                    .iter()
4328                    .map(|(_, v)| v.as_str())
4329                    .collect::<Vec<_>>()
4330                    .join(". "),
4331            ));
4332        }
4333        if n.kind == Kind::Cit {
4334            // The mark's own identity: the raw key. The entry is
4335            // one arrow away (`//cit--> ::`); rendered citation
4336            // styles are presentation, not structure.
4337            return Some(Value::Str(n.target.clone().unwrap_or_default()));
4338        }
4339        if n.kind == Kind::Anchor {
4340            // Invisible in the prose; its own projection is the
4341            // name it bears.
4342            return Some(Value::Str(n.onym.clone().unwrap_or_default()));
4343        }
4344        Some(Value::Str(n.prose.clone()))
4345    }
4346
4347    /// The edge label is the family name both ends share —
4348    /// `->footnote` / `->endnote` — the node's own kind, since
4349    /// resolution re-kinds an open-family callout to its body.
4350    fn links(&self, node: NodeId) -> Vec<(String, NodeId)> {
4351        if let Some(v) = tier::virt(node) {
4352            return self.v_links(v);
4353        }
4354        let n = &self.nodes[node.0 as usize];
4355        // The dictionary vocabulary: a usage label resolves against
4356        // the abbreviations list by its text.
4357        if n.kind == Kind::Span
4358            && n.span_kind.as_deref() == Some("label")
4359            && let Some(item) = self.abbreviation(&n.text)
4360        {
4361            return vec![("abbreviation".to_string(), item)];
4362        }
4363        // The literary reading: a mention reaches its character.
4364        let character: Vec<(String, NodeId)> = n
4365            .character
4366            .map(|c| ("character".to_string(), c))
4367            .into_iter()
4368            .chain(n.place.map(|p| ("place".to_string(), p)))
4369            .collect();
4370        // A token's dependency head (`->head`) and its enhanced
4371        // heads (`->ehead`, the relation at `$-::rel`), under a
4372        // CoNLL-U annotation.
4373        if !character.is_empty() && n.kind != Kind::Span {
4374            return character;
4375        }
4376        if let Some(t) = n.ref_edge {
4377            // The atrep rule: a resolved mention emits its typed
4378            // crosslink — `->ref`, or `->cit` for a citation.
4379            let label = if n.kind == Kind::Cit { "cit" } else { "ref" };
4380            return vec![(label.to_string(), t)];
4381        }
4382        // The literary reading: a span reaches the tokens it
4383        // covers (`//quotation<said>->token<word>`).
4384        if n.kind == Kind::Span && !(n.span_tokens.is_empty() && character.is_empty()) {
4385            return n
4386                .span_tokens
4387                .iter()
4388                .map(|&t| ("token".to_string(), t))
4389                .chain(character)
4390                .collect();
4391        }
4392        match n.note_edge {
4393            Some(body) => vec![(n.kind.name().unwrap_or("footnote").to_string(), body)],
4394            None => Vec::new(),
4395        }
4396    }
4397
4398    /// `$-::rel` on a dependency edge: the relation — `->head`'s
4399    /// is the dependent's `::deprel`, `->ehead`'s the enhanced
4400    /// relation to that head.
4401    fn link_property(
4402        &self,
4403        source: NodeId,
4404        label: &str,
4405        target: NodeId,
4406        name: &str,
4407    ) -> Option<Value> {
4408        if name != "rel" {
4409            return None;
4410        }
4411        if let Some(row) = ann::ann_row(source) {
4412            return self.a_link_property(row, label, target);
4413        }
4414        if tier::virt(source).is_some() {
4415            return None;
4416        }
4417        None
4418    }
4419
4420    fn backlinks(&self, node: NodeId) -> Vec<(String, NodeId)> {
4421        if let Some(row) = ann::ann_row(node) {
4422            return self.a_backlinks(row);
4423        }
4424        if tier::virt(node).is_some() {
4425            return Vec::new();
4426        }
4427        let n = &self.nodes[node.0 as usize];
4428        let mut out: Vec<(String, NodeId)> = n
4429            .ref_cites
4430            .iter()
4431            .map(|&c| {
4432                let label = if self.nodes[c.0 as usize].kind == Kind::Cit {
4433                    "cit"
4434                } else {
4435                    "ref"
4436                };
4437                (label.to_string(), c)
4438            })
4439            .collect();
4440        let label = n.kind.name().unwrap_or("footnote");
4441        out.extend(n.cites.iter().map(|&c| (label.to_string(), c)));
4442        let mention_label = if n.kind == Kind::Place {
4443            "place"
4444        } else {
4445            "character"
4446        };
4447        out.extend(n.mentions.iter().map(|&m| (mention_label.to_string(), m)));
4448        out
4449    }
4450
4451    /// `//ref-->`: land on the bearer — the block if a block
4452    /// bears the name, the point anchor if a point does. The
4453    /// text-level hints choose the reading: `-->block` normalizes
4454    /// a point landing to its enclosing block, `-->point` selects
4455    /// point targets only; bare is the exact landing.
4456    fn resolve(&self, node: NodeId, property: &str, hint: Option<&str>) -> Option<NodeId> {
4457        if tier::virt(node).is_some() {
4458            return None;
4459        }
4460        let n = &self.nodes[node.0 as usize];
4461        if n.kind == Kind::Span && n.span_kind.as_deref() == Some("label") {
4462            return (property == "abbreviation"
4463                && matches!(hint, None | Some("*") | Some("block")))
4464            .then(|| self.abbreviation(&n.text))
4465            .flatten();
4466        }
4467        if !matches!(n.kind, Kind::Ref | Kind::Cit) || property != "target" {
4468            return None;
4469        }
4470        let landed = n.ref_edge?;
4471        let is_point = self.nodes[landed.0 as usize].kind == Kind::Anchor;
4472        match hint {
4473            None | Some("*") => Some(landed),
4474            Some("point") => is_point.then_some(landed),
4475            Some("block") => {
4476                if !is_point {
4477                    return Some(landed);
4478                }
4479                // The anchor's enclosing block.
4480                let mut at = self.nodes[landed.0 as usize].parent;
4481                while let Some(b) = at {
4482                    let bn = &self.nodes[b.0 as usize];
4483                    if bn.kind != Kind::Document
4484                        && !matches!(bn.kind, Kind::Ref | Kind::Anchor | Kind::IndexMark)
4485                    {
4486                        return Some(b);
4487                    }
4488                    at = bn.parent;
4489                }
4490                None
4491            }
4492            Some(_) => None,
4493        }
4494    }
4495
4496    /// The bare arrow's property: a ref resolves its target.
4497    fn ref_property(&self, node: NodeId) -> Option<String> {
4498        if tier::virt(node).is_some() {
4499            return None;
4500        }
4501        let n = &self.nodes[node.0 as usize];
4502        if n.kind == Kind::Span && n.span_kind.as_deref() == Some("label") {
4503            return Some("abbreviation".to_string());
4504        }
4505        matches!(n.kind, Kind::Ref | Kind::Cit).then(|| "target".to_string())
4506    }
4507
4508    /// A ref the producer declared external: the absolute URL,
4509    /// relative targets joined against the document's own URL —
4510    /// the same acquisition rung the html DOM reading speaks.
4511    fn external_ref(
4512        &self,
4513        node: NodeId,
4514        property: &str,
4515        hint: Option<&str>,
4516    ) -> Option<quarb::ExternalRef> {
4517        if !matches!(hint, None | Some("*")) || tier::virt(node).is_some() {
4518            return None;
4519        }
4520        let n = &self.nodes[node.0 as usize];
4521        if n.kind != Kind::Ref || property != "target" || n.internal {
4522            return None;
4523        }
4524        let t = n.target.as_deref()?.trim();
4525        if t.is_empty() || t.starts_with('#') {
4526            return None;
4527        }
4528        let u = match url::Url::parse(t) {
4529            Ok(u) => u,
4530            Err(url::ParseError::RelativeUrlWithoutBase) => self
4531                .link_base
4532                .as_ref()
4533                .or(self.document_url.as_ref())?
4534                .join(t)
4535                .ok()?,
4536            Err(_) => return None,
4537        };
4538        if u.scheme() != "http" && u.scheme() != "https" {
4539            return None;
4540        }
4541        Some(quarb::ExternalRef::uri(u.as_str()))
4542    }
4543
4544    /// A sibling document's `#fragment` lands here: the bearer of
4545    /// that name — a labeled block, or a point anchor.
4546    fn resolve_fragment(&self, _node: NodeId, fragment: &str) -> Option<NodeId> {
4547        self.onyms.get(fragment).copied()
4548    }
4549
4550    /// Ruling #29: the text level's surface is the vocabulary
4551    /// itself — no document can introduce a property name — so
4552    /// its two annotations answer at `::` as well.
4553    fn aliased_metadata(&self, _node: NodeId) -> &'static [&'static str] {
4554        &["level", "lang", "form", "n"]
4555    }
4556
4557    /// `::::level` on sections (the source heading level) and
4558    /// `::::lang` on verbatim blocks (the declared language).
4559    fn metadata(&self, node: NodeId, key: &str) -> Option<Value> {
4560        if let Some(v) = tier::virt(node) {
4561            return self.v_metadata(v, key);
4562        }
4563        let n = &self.nodes[node.0 as usize];
4564        // The lossless layer: on the document node, every
4565        // declaration under the name it was declared with.
4566        if n.kind == Kind::Document
4567            && let Some(v) = self.head.declared_value(key)
4568        {
4569            return Some(v);
4570        }
4571        match key {
4572            "level" => n.level.map(|l| Value::Int(l as i64)),
4573            // A verbatim block's declared language; on the
4574            // literary reading, the genos after `foreign`
4575            // (`@/…/@.foreign.la`).
4576            "lang" => n
4577                .lang
4578                .clone()
4579                .or_else(|| {
4580                    let i = n.genoses.iter().position(|g| g == "foreign")?;
4581                    n.genoses.get(i + 1).cloned()
4582                })
4583                .map(Value::Str),
4584            "at" if n.at.is_some() => n.at.map(|(lo, _)| Value::Int(lo as i64)),
4585            // The corpus reading: a token's position and sentence.
4586            // A sentence node's document-wide ordinal, under both
4587            // spellings, so `/paragraph[7]/sentence[3]::::n` and a
4588            // token's `::::sentence` agree.
4589            "n" | "sentence" if n.kind == Kind::Sentence => n.taxis.map(Value::Int),
4590            // A stored sentence's coordinates: its first row's
4591            // offset and its last row's end in the block's text (a
4592            // treebank's own sentence is the block: 0 and its length).
4593            "at" | "end" if n.kind == Kind::Sentence => {
4594                let rows = self.ann_of.get(&(node.0 as usize))?;
4595                let (first, last) = (*rows.first()?, *rows.last()?);
4596                let block = self.ann[first as usize].block as usize;
4597                if block == node.0 as usize {
4598                    return Some(Value::Int(if key == "at" {
4599                        0
4600                    } else {
4601                        n.prose.chars().count() as i64
4602                    }));
4603                }
4604                Some(Value::Int(if key == "at" {
4605                    self.chars_to(block, self.ann[first as usize].at)
4606                } else {
4607                    let r = &self.ann[last as usize];
4608                    self.chars_to(block, r.at) + self.ann_names.get(r.form).chars().count() as i64
4609                }))
4610            }
4611            // `::::resolved` on a callout: the broken-apparatus
4612            // linter's fact.
4613            "resolved" if n.deixis => Some(Value::Bool(!n.dangling)),
4614            // …and on an internal ref: the broken-cross-reference
4615            // linter's fact. External refs resolve at query time,
4616            // so the fact is not theirs to answer.
4617            "resolved" if n.kind == Kind::Cit || (n.kind == Kind::Ref && n.internal) => {
4618                Some(Value::Bool(n.ref_edge.is_some()))
4619            }
4620            // The declared spelling was a margin form (a Tufte
4621            // sidenote): family footnote, placement preserved.
4622            "form" if n.margin => Some(Value::Str("margin".to_string())),
4623            // The entry's genus, also queryable as an annotation.
4624            "genus" if n.kind == Kind::Bib => n.genus.clone().map(Value::Str),
4625            _ => None,
4626        }
4627    }
4628}