Skip to main content

carve/
includes.rs

1//! Processor-level file inclusion (`{{ path }}`), spec PART 9 §19 (I1-I11).
2//!
3//! The core parser is untouched and performs NO file I/O: it does not know the
4//! directive exists and leaves `{{ … }}` as ordinary inline text. Expansion is
5//! this separate, opt-in pass over an already-parsed [`Document`]. With no
6//! resolver configured the directive stays literal, which is the behavior the
7//! conformance corpus pins.
8//!
9//! ```
10//! use carve::{expand_includes, parse, render_html, IncludeDenial, IncludeOptions, IncludeResolved};
11//!
12//! let source = "Before.\n\n{{ child.crv }}\n";
13//! let doc = parse(source);
14//! let resolver = |path: &str, _ctx: &carve::IncludeContext<'_>| {
15//!     (path == "child.crv")
16//!         .then(|| IncludeResolved::from("Included body."))
17//!         .ok_or(IncludeDenial::NotFound)
18//! };
19//! let opts = IncludeOptions::new().with_resolver(&resolver);
20//! let result = expand_includes(doc, source, &opts);
21//! assert!(render_html(&result.doc).unwrap().contains("<p>Included body.</p>"));
22//! ```
23
24use std::collections::{BTreeMap, HashMap, HashSet};
25// Only the filesystem resolver deals in paths; nothing else in the pass does.
26#[cfg(feature = "fs")]
27use std::path::{Path, PathBuf};
28
29use crate::ast::{
30    AttrSlot, Attrs, BlockNode, Document, FigureTarget, Heading, Image, InlineNode, Paragraph,
31};
32use crate::extension::CarveExtension;
33use crate::include_walk::SubtreeVisitor;
34use crate::parse::slugify_parse;
35use crate::render::plain_inlines;
36
37/// Default transitive include depth limit (spec I6 recommends at least 16).
38pub const DEFAULT_MAX_DEPTH: usize = 16;
39
40/// Resolver calls allowed for one expansion.
41///
42/// The byte budget bounds expanded OUTPUT; it does not bound the WORK done to
43/// produce it, because a target is resolved before its size is known and a
44/// document may carry one directive per dozen bytes. Section 19 requires a
45/// finite bound here, and without one a megabyte of directives was tens of
46/// thousands of file reads for a bounded expansion.
47pub const DEFAULT_MAX_RESOLVER_CALLS: usize = 1000;
48
49/// Include warnings retained for one expansion.
50///
51/// Warnings are per-directive, so a document of refused directives otherwise
52/// allocates one per directive. One warning per distinct rule always survives
53/// the cap, and the suppressed count says how many did not.
54pub const DEFAULT_MAX_WARNINGS: usize = 100;
55
56fn spent_message(rule: &str, path: &str) -> String {
57    if rule == "include-call-limit" {
58        format!("Include resolver call limit exceeded for \"{path}\".")
59    } else {
60        format!("Include byte budget exceeded by \"{path}\".")
61    }
62}
63/// Byte-budget floor, matching the §25 amplification bound `max(1 MB, 8 x input)`.
64const MIN_BUDGET: usize = 1024 * 1024;
65
66/// A degradation or rename reported by [`expand_includes`].
67///
68/// Every failure mode in spec I7 produces one of these and leaves the offending
69/// directive LITERAL; inclusion never silently drops a directive.
70///
71/// Positions are 1-based, in the file attributed by [`Self::file`]. A warning
72/// with no positioned node defaults to line 1, column 1, as in carve-js. Full
73/// position remapping (spec I4) is out of scope in every engine.
74#[derive(Debug, Clone, PartialEq, Eq)]
75pub struct IncludeWarning {
76    /// Line where the reported directive or node starts.
77    pub line: usize,
78    /// Column where the reported directive or node starts.
79    pub column: usize,
80    /// Stable rule id, e.g. `"include-cycle"`.
81    pub rule: String,
82    /// Human-readable explanation.
83    pub message: String,
84    /// Identity of the file the warning AROSE in: the canonical id a resolver
85    /// returned for that file, or the raw directive path when the resolver
86    /// returned plain source. A directive that failed to resolve is attributed
87    /// to the document CONTAINING it, not to the target it names; a warning
88    /// raised while expanding a child (a heading clamp, a rename, a nested
89    /// cycle) is attributed to that child.
90    ///
91    /// `None` for a top-level document the caller gave no `source_path` for -
92    /// there is no identity to report, and none is invented.
93    pub file: Option<String>,
94    /// Directive sites that led from the root to this warning.
95    pub included_by: Vec<IncludeSite>,
96}
97
98/// Source location of one directive in an include chain.
99#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
100pub struct IncludeSite {
101    /// Identity of the file containing the directive, or an empty string when
102    /// the root document has no source path.
103    pub file: String,
104    /// Source line of the directive, counted from 1.
105    pub line: usize,
106    /// Source column of the directive, counted from 1.
107    pub column: usize,
108}
109
110/// Included content admitted by the expansion pass.
111#[derive(Debug, Clone)]
112pub enum AcceptedInclude {
113    /// Carve source inserted as parsed document content.
114    Carve {
115        /// Canonical file identity or the directive path when none was supplied.
116        file: String,
117        /// Directive sites leading to this inclusion.
118        included_by: Vec<IncludeSite>,
119        /// Selected Carve source text.
120        source: String,
121        /// Lines skipped before the selection; zero for the whole file.
122        first_line: usize,
123        /// Line range of the selected section, when one was named.
124        selected_lines: Option<(usize, usize)>,
125    },
126    /// Source inserted as a code block.
127    Code {
128        /// Canonical file identity or the directive path when none was supplied.
129        file: String,
130        /// Directive sites leading to this inclusion.
131        included_by: Vec<IncludeSite>,
132        /// Code block, or table converted from delimited data.
133        block: Box<BlockNode>,
134        /// Code payload before insertion.
135        payload: String,
136    },
137}
138
139/// Context handed to a resolver for one directive.
140#[derive(Debug, Clone)]
141pub struct IncludeContext<'a> {
142    /// Identity of the root document, when the host supplied one.
143    pub source_path: Option<&'a str>,
144    /// Include chain, root first. Each entry is the canonical id a resolver
145    /// returned, or the raw directive path when it returned plain source.
146    /// The last entry is the file containing the directive being resolved,
147    /// which is what relative resolution keys off (spec I1).
148    pub stack: &'a [String],
149    /// Zero-based include depth of the directive being resolved.
150    pub depth: usize,
151}
152
153/// What a resolver produces: source text, optionally with a canonical id.
154///
155/// The id feeds cycle detection (spec I6) and dependency identity (I11), and
156/// becomes the parent entry in [`IncludeContext::stack`] for nested resolves.
157/// Resolvers that map paths to files (filesystem, VFS) SHOULD supply one;
158/// without it two spellings of the same file (`b.crv` vs `./b.crv`) defeat the
159/// cycle guard and only the depth limit stops the recursion.
160#[derive(Debug, Clone, PartialEq, Eq)]
161pub struct IncludeResolved {
162    /// Source text returned by the resolver.
163    pub source: String,
164    /// Canonical identity of that source, if known.
165    pub id: Option<String>,
166}
167
168impl IncludeResolved {
169    /// Source text with a canonical id.
170    pub fn with_id(source: impl Into<String>, id: impl Into<String>) -> Self {
171        Self {
172            source: source.into(),
173            id: Some(id.into()),
174        }
175    }
176}
177
178impl From<String> for IncludeResolved {
179    fn from(source: String) -> Self {
180        Self { source, id: None }
181    }
182}
183
184impl From<&str> for IncludeResolved {
185    fn from(source: &str) -> Self {
186        Self {
187            source: source.to_string(),
188            id: None,
189        }
190    }
191}
192
193/// Why a resolver refused, in the classes PART 9 section 19 names.
194///
195/// A refusal is still an attempted dependency whatever its class: a host wants
196/// to re-check any of them if the tree changes.
197#[derive(Debug, Clone, Copy, PartialEq, Eq)]
198pub enum IncludeDenial {
199    /// The target resolves outside the configured root.
200    OutsideRoot,
201    /// Nothing is there to read.
202    NotFound,
203    /// No root is configured, so containment cannot be decided.
204    NoRoot,
205    /// A policy of the resolver's own refused it: an absolute path where those
206    /// are not allowed, a file past a size cap, an unreadable target.
207    Denied,
208    /// The target was read but its bytes are not well-formed UTF-8.
209    /// Like NUL content, this leaves no denial class in the dependency report.
210    NonText,
211    /// Refused for a reason the resolver does not classify.
212    Unresolved,
213}
214
215impl IncludeDenial {
216    /// The portable class name (spec section 19).
217    pub fn as_str(self) -> &'static str {
218        match self {
219            Self::OutsideRoot => "outside-root",
220            Self::NotFound => "not-found",
221            Self::NoRoot => "no-root",
222            Self::Denied => "include-denied",
223            Self::NonText => "include-denied",
224            Self::Unresolved => "include-unresolved",
225        }
226    }
227}
228
229/// Host-supplied path resolution (spec I3). The core never touches the
230/// filesystem; a host that wants inclusion supplies one of these.
231///
232/// The error carries the class, which reaches the caller on the attempted
233/// dependency ([`IncludeDependency::denial`]).
234pub trait IncludeResolver {
235    /// Resolve one directive path in its containing document context.
236    fn resolve(
237        &self,
238        path: &str,
239        ctx: &IncludeContext<'_>,
240    ) -> Result<IncludeResolved, IncludeDenial>;
241
242    /// Where a refused target WOULD appear, when this resolver can say (spec
243    /// I11). The id reaches the caller on the attempted dependency, so a host
244    /// watches that path and rebuilds when the file arrives. `None` leaves the
245    /// dependency spelled as the directive wrote it.
246    fn unresolved_id(&self, _path: &str, _ctx: &IncludeContext<'_>) -> Option<String> {
247        None
248    }
249}
250
251impl<F> IncludeResolver for F
252where
253    F: Fn(&str, &IncludeContext<'_>) -> Result<IncludeResolved, IncludeDenial>,
254{
255    fn resolve(
256        &self,
257        path: &str,
258        ctx: &IncludeContext<'_>,
259    ) -> Result<IncludeResolved, IncludeDenial> {
260        self(path, ctx)
261    }
262}
263
264/// Expansion knobs. Default-constructed options carry NO resolver, so
265/// [`expand_includes`] is a no-op that leaves every directive literal.
266#[derive(Default, Clone)]
267pub struct IncludeOptions<'a> {
268    resolver: Option<&'a dyn IncludeResolver>,
269    source_path: Option<String>,
270    max_depth: Option<usize>,
271    max_bytes: Option<usize>,
272    max_resolver_calls: Option<usize>,
273    max_warnings: Option<usize>,
274    extensions: Vec<&'a dyn CarveExtension>,
275    collect_included: bool,
276}
277
278impl<'a> IncludeOptions<'a> {
279    /// Create include options without a resolver.
280    pub fn new() -> Self {
281        Self::default()
282    }
283
284    /// Retain accepted include records in the result.
285    pub fn collecting_included(mut self) -> Self {
286        self.collect_included = true;
287        self
288    }
289
290    /// Supply the host's path resolver.
291    pub fn with_resolver(mut self, resolver: &'a dyn IncludeResolver) -> Self {
292        self.resolver = Some(resolver);
293        self
294    }
295
296    /// Identity of the root document. Seeds the cycle-guard stack and the
297    /// `file` attribution on warnings raised in the root.
298    pub fn with_source_path(mut self, path: impl Into<String>) -> Self {
299        self.source_path = Some(path.into());
300        self
301    }
302
303    /// Maximum transitive include depth (spec I6). Defaults to
304    /// [`DEFAULT_MAX_DEPTH`].
305    pub fn with_max_depth(mut self, depth: usize) -> Self {
306        self.max_depth = Some(depth);
307        self
308    }
309
310    /// Total expanded-source byte budget. Defaults to `max(1 MB, 8 x root
311    /// source bytes)`, the §25 amplification bound.
312    pub fn with_max_bytes(mut self, bytes: usize) -> Self {
313        self.max_bytes = Some(bytes);
314        self
315    }
316
317    /// Resolver calls allowed for one expansion. Defaults to
318    /// [`DEFAULT_MAX_RESOLVER_CALLS`].
319    ///
320    /// This is the bound on the pass's own WORK, which the byte budget does not
321    /// give: a target is resolved before its size is known.
322    pub fn with_max_resolver_calls(mut self, calls: usize) -> Self {
323        self.max_resolver_calls = Some(calls);
324        self
325    }
326
327    /// Include warnings retained. Defaults to [`DEFAULT_MAX_WARNINGS`]. One per
328    /// distinct rule always survives, and the result reports how many did not.
329    pub fn with_max_warnings(mut self, warnings: usize) -> Self {
330        self.max_warnings = Some(warnings);
331        self
332    }
333
334    /// Parse each child with this extension too. Pass the set the PARENT was
335    /// parsed with, so the same text means the same thing in either file.
336    /// [`crate::prepare_doc_with_includes`] forwards its own parse options' set
337    /// and does not read this one.
338    pub fn with_extension(mut self, extension: &'a dyn CarveExtension) -> Self {
339        self.extensions.push(extension);
340        self
341    }
342}
343
344/// One include target touched during expansion (spec I11).
345///
346/// Unresolved targets are reported too: a host that watched only the files it
347/// successfully read would never learn that a previously-missing target now
348/// EXISTS, so a preview would stay stale at exactly the moment the author
349/// fixes the problem.
350#[derive(Debug, Clone, PartialEq, Eq)]
351pub struct IncludeDependency {
352    /// The resolver's canonical id when it supplied one (the identity the
353    /// cycle guard uses), otherwise the directive path as written.
354    pub id: String,
355    /// True when the target's SOURCE WAS READ - and nothing more (spec I11).
356    ///
357    /// It is NOT a report on whether the content was merged. A target that was
358    /// read and then rejected (a missing `#section`, a cycle, an exhausted
359    /// byte budget) stays `true`, because the file exists and a host must keep
360    /// watching it: editing the child to fix the problem has to invalidate the
361    /// preview. Only a target whose source was never obtained - unresolvable,
362    /// denied by containment, or refused before any read at the depth limit -
363    /// is `false`.
364    pub resolved: bool,
365    /// Why the resolver refused, when it refused and said so. `None` for a
366    /// dependency that resolved, and for a refusal the engine made itself.
367    pub denial: Option<IncludeDenial>,
368}
369
370/// Outcome of [`expand_includes`].
371#[derive(Debug, Clone)]
372pub struct IncludeResult {
373    /// Document after accepted includes were expanded.
374    pub doc: Document,
375    /// Retained diagnostics from expansion.
376    pub warnings: Vec<IncludeWarning>,
377    /// Warnings raised but not retained once the cap was reached. Zero on every
378    /// uncapped run; non-zero means `warnings` is a sample, not the whole
379    /// report, so a capped one is never mistaken for a clean one.
380    pub suppressed_warnings: usize,
381    /// Every include target touched during the whole recursive expansion,
382    /// de-duplicated, in first-encounter order. Empty without a resolver.
383    pub dependencies: Vec<IncludeDependency>,
384    /// Bytes CHARGED against the byte budget: the size of every target the
385    /// resolver handed back, whether or not expansion went on to admit it.
386    ///
387    /// Published because the budget's own arithmetic is otherwise unobservable
388    /// from outside, and the conformance corpus pins it. PART 9 section 19 is
389    /// explicit that the budget bounds the expanded OUTPUT and not the WORK -
390    /// a target is resolved before its size is known - so a counter of bytes
391    /// ADMITTED could not tell "read nothing" from "read a file and refused
392    /// it", which is the one thing a reader of this number wants to know.
393    pub charged_bytes: usize,
394    /// Accepted content records when collection was enabled.
395    pub included: Vec<AcceptedInclude>,
396}
397
398// ---------------------------------------------------------------------------
399// Directive syntax (spec I1 / PART 6)
400// ---------------------------------------------------------------------------
401
402#[derive(Debug, Clone, PartialEq, Eq)]
403enum Shift {
404    By(i32),
405    Auto,
406}
407
408#[derive(Debug, Clone, PartialEq, Eq)]
409struct Directive {
410    path: String,
411    section: Option<String>,
412    lines: Option<(usize, usize)>,
413    rows: Option<(usize, usize)>,
414    cols: Option<IncludeColumns>,
415    table_attrs: Option<Attrs>,
416    region: Option<String>,
417    code: Option<Option<String>>,
418    shift: Shift,
419}
420
421#[derive(Debug, Clone, PartialEq, Eq)]
422enum IncludeColumns {
423    Names(Vec<String>),
424    Range(usize, usize),
425}
426
427/// A `{table}` include reads a well-formed region marker line as no record (I1d).
428fn table_include(
429    code: &crate::ast::CodeBlock,
430    payload: &[crate::parse::delimited_table::PayloadLine],
431) -> Option<crate::ast::Table> {
432    crate::parse::delimited_table::from_code_without(code, payload, is_table_record_marker)
433}
434
435fn selected_table_include(d: &Directive) -> bool {
436    d.table_attrs
437        .as_ref()
438        .is_some_and(|attrs| attrs.key_values.get("table").is_some_and(String::is_empty))
439        && (d.rows.is_some() || d.cols.is_some() || d.lines.is_some() || d.region.is_some())
440}
441
442fn is_ident_start(c: char) -> bool {
443    c.is_ascii_alphabetic() || c == '_'
444}
445
446/// PART 6 `include_section` takes an `explicit_identifier`, which may also open
447/// on a digit (`#2024-plan`).
448fn is_explicit_ident_start(c: char) -> bool {
449    c.is_ascii_alphanumeric() || c == '_'
450}
451
452fn is_ident_rest(c: char) -> bool {
453    c.is_ascii_alphanumeric() || c == '_' || c == '-'
454}
455
456/// Bare path: stops at space, `#`, `@`, `}` (PART 6), plus the quote
457/// characters that introduce the quoted form.
458fn is_bare_path_char(c: char) -> bool {
459    !matches!(c, '#' | '@' | '}' | '"' | '\u{201c}') && !matches!(c, ' ' | '\n' | '\r')
460}
461
462/// Match a directive starting exactly at `start`, returning the byte index one
463/// past the closing `}}` plus the parsed shape.
464///
465/// `{{`, whitespace, a bare / straight- or curly-quoted path, an optional
466/// `#section`, an option tail, at least one whitespace, `}}`. THE CLOSER is
467/// the first such `}}` that falls OUTSIDE any quoted run (PART 6): a
468/// `quoted_include_path` or an option's `quoted_value` may carry the pair.
469fn match_directive_at(text: &str, start: usize) -> Option<(usize, RawDirective)> {
470    let bytes = text.as_bytes();
471    if !text[start..].starts_with("{{") {
472        return None;
473    }
474    let mut i = start + 2;
475    // `\s+` after the opening braces.
476    let ws_start = i;
477    while i < bytes.len() && bytes[i] == b' ' {
478        i += text[i..].chars().next()?.len_utf8();
479    }
480    if i == ws_start {
481        return None;
482    }
483
484    // Path: straight-quoted, curly-quoted (smart typography already rewrote
485    // the source), or bare.
486    let rest = &text[i..];
487    let (path, after_path) = if let Some(body) = rest.strip_prefix('"') {
488        // `"((?:\\.|[^"\\])*)"`: a backslash escapes the following character.
489        // It decodes before ASCII punctuation, as in a quoted attribute value
490        // (carve#2778); every other pair stays verbatim.
491        let mut out = String::new();
492        let mut chars = body.char_indices();
493        let mut close = None;
494        while let Some((off, c)) = chars.next() {
495            match c {
496                '\\' => {
497                    let (_, next) = chars.next()?;
498                    if next.is_ascii_punctuation() {
499                        out.push(next);
500                    } else {
501                        out.push('\\');
502                        out.push(next);
503                    }
504                }
505                '"' => {
506                    close = Some(off);
507                    break;
508                }
509                _ => out.push(c),
510            }
511        }
512        (out, i + 1 + close? + 1)
513    } else if let Some(body) = rest.strip_prefix('\u{201c}') {
514        let close = body.find('\u{201d}')?;
515        (
516            body[..close].to_string(),
517            i + '\u{201c}'.len_utf8() + close + '\u{201d}'.len_utf8(),
518        )
519    } else {
520        let end = rest
521            .char_indices()
522            .find(|(_, c)| !is_bare_path_char(*c))
523            .map(|(o, _)| o)
524            .unwrap_or(rest.len());
525        if end == 0 {
526            return None;
527        }
528        (rest[..end].to_string(), i + end)
529    };
530    i = after_path;
531
532    // A quoted form can spell an empty or whitespace-only path (`{{ "" }}`,
533    // `{{ "   " }}`); the maintainer ruled such a token is NOT a directive at
534    // all. Rejecting it HERE is what converges the serializer and the expander
535    // on one recognizer: the spec requires the serializer's preserved set and
536    // the expander's recognized set to be identical, and every recognition
537    // path in this module funnels through `match_directive_at`. Bare paths
538    // cannot be empty or hold whitespace, so this only ever fires on the
539    // quoted forms.
540    if path.trim().is_empty() {
541        return None;
542    }
543
544    // Optional `#section`, exactly one, after the path. The separating
545    // whitespace is OPTIONAL (carve#2773): `bare_include_path` already stops at
546    // `#`, so requiring a run left `{{ c.crv#Alpha }}` unrecognized and the
547    // whole directive literal with NO diagnostic, which is the one outcome §19
548    // forbids.
549    let mut section = None;
550    {
551        let tail = &text[i..];
552        let ws = tail.len() - tail.trim_start_matches(' ').len();
553        {
554            let after_ws = &tail[ws..];
555            if let Some(name) = after_ws.strip_prefix('#') {
556                let mut it = name.chars();
557                if let Some(first) = it.next() {
558                    if is_explicit_ident_start(first) {
559                        let end = name
560                            .char_indices()
561                            .find(|(_, c)| !is_ident_rest(*c))
562                            .map(|(o, _)| o)
563                            .unwrap_or(name.len());
564                        section = Some(name[..end].to_string());
565                        i += ws + 1 + end;
566                    }
567                }
568            }
569        }
570    }
571
572    // Option tail closed by the first `\s+}}` OUTSIDE any quoted run. A
573    // `quoted_value` (PART 4) excludes only its own quote, the backslash and
574    // the newline, so a `}}` between the quotes belongs to the run; an
575    // UNTERMINATED quote opens no run, and the first `}}` wins again.
576    //
577    // One forward pass, never a rewind: the first candidate passed while a run
578    // is PROVISIONALLY open is remembered, so an unterminated run already has
579    // its fallback in hand. Matching the quote to its closer first and then
580    // searching for `}}` is what would rescan the run once per opener.
581    let tail = &text[i..];
582    let tail_bytes = tail.as_bytes();
583    // A `}}` whose preceding character is whitespace, i.e. `whitespace+, "}}"`.
584    let is_closer = |j: usize| {
585        tail_bytes[j] == b'}'
586            && tail_bytes.get(j + 1) == Some(&b'}')
587            && tail[..j].chars().next_back().is_some_and(|c| c == ' ')
588    };
589    let mut j = 0usize;
590    let mut open_quote: Option<u8> = None;
591    let mut fallback: Option<usize> = None;
592    let mut saw_lines = false;
593    let hit = loop {
594        if j >= tail_bytes.len() {
595            break fallback?;
596        }
597        let c = tail_bytes[j];
598        let Some(quote) = open_quote else {
599            if c == b'@' && tail[j..].starts_with("@lines:") {
600                saw_lines = true;
601            }
602            // A separate nested opener before the line selector makes this
603            // candidate ineligible. Stop here rather than rescan to the same
604            // distant closer once for every opener in a malformed run.
605            if c == b'{'
606                && tail_bytes.get(j + 1) == Some(&b'{')
607                && j > 0
608                && tail_bytes[j - 1] == b' '
609                && !saw_lines
610            {
611                return None;
612            }
613            if c == b'"' || c == b'\'' {
614                open_quote = Some(c);
615            } else if is_closer(j) {
616                break j;
617            }
618            j += 1;
619            continue;
620        };
621        if c == b'\\' {
622            // `escaped_char`: the next character cannot close the run.
623            j += 2;
624        } else if c == quote {
625            // The run closed, so every candidate inside it belonged to it.
626            open_quote = None;
627            fallback = None;
628            j += 1;
629        } else if c == b'\n' {
630            // A quoted value stops at the newline, so no run ever opened.
631            if let Some(f) = fallback {
632                break f;
633            }
634            open_quote = None;
635        } else {
636            if fallback.is_none() && is_closer(j) {
637                fallback = Some(j);
638            }
639            j += 1;
640        }
641    };
642
643    let before = &tail[..hit];
644    Some((
645        i + hit + 2,
646        RawDirective {
647            path,
648            section,
649            options: before.trim_end_matches(' ').to_string(),
650        },
651    ))
652}
653
654struct RawDirective {
655    path: String,
656    section: Option<String>,
657    options: String,
658}
659
660/// Outcome of turning a raw match into a directive: options are validated here
661/// so an unknown key or malformed value degrades to a warning + literal (I1/I7).
662enum ParsedDirective {
663    Ok(Box<Directive>),
664    /// A recognized line selector with a malformed range. Keep its path so
665    /// the rejected directive can still record a resolved dependency.
666    InvalidLines(String),
667    InvalidTable(String, &'static str),
668    /// An `@…`-shaped option that is unknown or malformed; carries the offending token.
669    BadOption(String),
670    /// Not directive-shaped at all; no warning, stays literal silently.
671    NotADirective,
672    /// A second `#section` where the grammar holds one. Carries the offending
673    /// token.
674    DuplicateSection(String),
675    /// An option key given a second time (I1). Carries the offending token.
676    DuplicateOption(String),
677    RepeatedTableOption(String, String),
678}
679
680fn include_range(value: &str) -> Option<(usize, usize)> {
681    let (start, end) = value.split_once('-')?;
682    let positive =
683        |s: &str| !s.is_empty() && !s.starts_with('0') && s.bytes().all(|b| b.is_ascii_digit());
684    if !positive(start) || !positive(end) {
685        return None;
686    }
687    let (start, end) = (start.parse::<usize>().ok()?, end.parse::<usize>().ok()?);
688    (start <= end).then_some((start, end))
689}
690
691fn include_column_names(value: &str) -> Option<Vec<String>> {
692    let inner = value.strip_prefix('"')?.strip_suffix('"')?;
693    let mut names = Vec::new();
694    let mut current = String::new();
695    let mut chars = inner.chars().peekable();
696    while let Some(ch) = chars.next() {
697        match ch {
698            // `quoted_value`: a backslash escapes ASCII punctuation only (carve#4571).
699            '\\' if chars.peek().is_some_and(char::is_ascii_punctuation) => {
700                current.push(chars.next()?);
701            }
702            '"' => return None,
703            ',' => names.push(std::mem::take(&mut current)),
704            _ => current.push(ch),
705        }
706    }
707    names.push(current);
708    names.iter().all(|name| !name.is_empty()).then_some(names)
709}
710
711/// The characters of a quoted option value, with PART 4's escapes applied.
712fn unquote_option_value(value: &str) -> Option<String> {
713    let quote = value.chars().next().filter(|c| matches!(c, '"' | '\''))?;
714    let inner = value.strip_prefix(quote)?.strip_suffix(quote)?;
715    let mut out = String::with_capacity(inner.len());
716    let mut chars = inner.chars().peekable();
717    while let Some(ch) = chars.next() {
718        match ch {
719            '\\' if chars.peek().is_some_and(char::is_ascii_punctuation) => out.push(chars.next()?),
720            c if c == quote => return None,
721            c => out.push(c),
722        }
723    }
724    Some(out)
725}
726
727/// A `#name` the section slot would have taken, had it not already been full.
728fn is_section_token(part: &str) -> bool {
729    let Some(name) = part.strip_prefix('#') else {
730        return false;
731    };
732    let mut it = name.chars();
733    it.next().is_some_and(is_explicit_ident_start) && it.all(is_ident_rest)
734}
735
736/// The option tail split into tokens. A token ends at whitespace or at an `@`
737/// outside a quoted run, since the whitespace before an option is optional
738/// (carve#2773): `@shift:1@lines:1-8` is two options.
739fn option_parts(tail: &str) -> Vec<&str> {
740    let mut parts = Vec::new();
741    let mut start: Option<usize> = None;
742    let mut quote: Option<char> = None;
743    let mut escaped = false;
744    for (i, c) in tail.char_indices() {
745        if let Some(q) = quote {
746            if escaped {
747                escaped = false;
748            } else if c == '\\' {
749                escaped = true;
750            } else if c == q {
751                quote = None;
752            }
753            continue;
754        }
755        if c == ' ' {
756            if let Some(s) = start.take() {
757                parts.push(&tail[s..i]);
758            }
759            continue;
760        }
761        if c == '@' {
762            if let Some(s) = start.filter(|&s| s < i) {
763                parts.push(&tail[s..i]);
764            }
765            start = Some(i);
766            continue;
767        }
768        if c == '"' || c == '\'' {
769            quote = Some(c);
770        }
771        start.get_or_insert(i);
772    }
773    if let Some(s) = start {
774        parts.push(&tail[s..]);
775    }
776    parts
777}
778
779fn parse_options(raw: RawDirective) -> ParsedDirective {
780    let mut lines = None;
781    let mut rows = None;
782    let mut cols = None;
783    let mut region = None;
784    let mut code = None;
785    let mut shift = Shift::By(0);
786    let mut seen: HashSet<&str> = HashSet::new();
787    for part in option_parts(&raw.options) {
788        let bad = || {
789            if part.starts_with('@') {
790                ParsedDirective::BadOption(part.to_string())
791            } else {
792                ParsedDirective::NotADirective
793            }
794        };
795        // `include_section` is ONE optional slot, so a second name is not a
796        // directive the author can mean. Dropping it silently returned the
797        // fragment the author did not name with nothing on the page to say so.
798        if raw.section.is_some() && is_section_token(part) {
799            return ParsedDirective::DuplicateSection(part.to_string());
800        }
801        let Some(body) = part.strip_prefix('@') else {
802            return bad();
803        };
804        if body == "code" {
805            if !seen.insert("code") {
806                return ParsedDirective::DuplicateOption(part.to_string());
807            }
808            code = Some(None);
809            continue;
810        }
811        let Some((key, value)) = body.split_once(':') else {
812            if body == "rows" || body == "cols" {
813                return ParsedDirective::InvalidTable(
814                    raw.path,
815                    if body == "rows" {
816                        "include-rows-invalid"
817                    } else {
818                        "include-cols-invalid"
819                    },
820                );
821            }
822            return bad();
823        };
824        // PART 10: outside `@cols` quotes only delimit a value, so
825        // `@lines:"1-1"` reads as `@lines:1-1`.
826        let unquoted = (key != "cols")
827            .then(|| unquote_option_value(value))
828            .flatten();
829        let value = unquoted.as_deref().unwrap_or(value);
830        let key_ok = {
831            let mut it = key.chars();
832            it.next().is_some_and(is_ident_start) && it.all(is_ident_rest)
833        };
834        let value_ok = !value.is_empty()
835            && (key == "cols" && value.starts_with('"')
836                || value
837                    .chars()
838                    .all(|c| !matches!(c, '@' | '}') && !c.is_whitespace()));
839        if key == "lines" && !value_ok {
840            return ParsedDirective::InvalidLines(raw.path);
841        }
842        if (key == "rows" || key == "cols") && !value_ok {
843            return ParsedDirective::InvalidTable(
844                raw.path,
845                if key == "rows" {
846                    "include-rows-invalid"
847                } else {
848                    "include-cols-invalid"
849                },
850            );
851        }
852        if !key_ok || !value_ok {
853            return bad();
854        }
855        // I1: a key given twice is an error, never "last one wins". Checked
856        // after the value, so a malformed repeat still reports as malformed.
857        let repeated = || ParsedDirective::DuplicateOption(part.to_string());
858        match key {
859            "rows" => {
860                if !seen.insert("rows") {
861                    return ParsedDirective::RepeatedTableOption(raw.path, part.to_string());
862                }
863                rows = match include_range(value) {
864                    Some(range) => Some(range),
865                    None => return ParsedDirective::InvalidTable(raw.path, "include-rows-invalid"),
866                };
867            }
868            "cols" => {
869                if !seen.insert("cols") {
870                    return ParsedDirective::RepeatedTableOption(raw.path, part.to_string());
871                }
872                cols = match include_range(value)
873                    .map(|(a, b)| IncludeColumns::Range(a, b))
874                    .or_else(|| include_column_names(value).map(IncludeColumns::Names))
875                {
876                    Some(columns) => Some(columns),
877                    None => return ParsedDirective::InvalidTable(raw.path, "include-cols-invalid"),
878                };
879            }
880            "code" => {
881                if !value.chars().all(|c| {
882                    c.is_ascii_alphanumeric() || matches!(c, '_' | '+' | '#' | '.' | '/' | '-')
883                }) {
884                    return bad();
885                }
886                if !seen.insert("code") {
887                    return repeated();
888                }
889                code = Some(Some(value.to_string()));
890            }
891            "lines" => {
892                let Some((a_raw, b_raw)) = value.split_once('-') else {
893                    return ParsedDirective::InvalidLines(raw.path);
894                };
895                // `[1-9]\d*` on both sides: 1-based, so a leading zero is
896                // malformed rather than silently normalized.
897                let positive = |s: &str| {
898                    !s.is_empty()
899                        && !s.starts_with('0')
900                        && s.chars().all(|c| c.is_ascii_digit())
901                        && s.parse::<usize>().is_ok()
902                };
903                if !positive(a_raw) || !positive(b_raw) {
904                    return ParsedDirective::InvalidLines(raw.path);
905                }
906                let (a, b) = (
907                    a_raw.parse::<usize>().unwrap_or(0),
908                    b_raw.parse::<usize>().unwrap_or(0),
909                );
910                // An inverted range is an error, not an empty selection.
911                if b < a {
912                    return ParsedDirective::InvalidLines(raw.path);
913                }
914                if !seen.insert("lines") {
915                    return repeated();
916                }
917                lines = Some((a, b));
918            }
919            "region" => {
920                let mut chars = value.chars();
921                if !chars.next().is_some_and(is_ident_start) || !chars.all(is_ident_rest) {
922                    return bad();
923                }
924                if !seen.insert("region") {
925                    return repeated();
926                }
927                region = Some(value.to_string());
928            }
929            "shift" => {
930                let parsed = if value == "auto" {
931                    Shift::Auto
932                } else {
933                    let digits = value.strip_prefix(['+', '-']).unwrap_or(value);
934                    if digits.is_empty() || !digits.chars().all(|c| c.is_ascii_digit()) {
935                        return bad();
936                    }
937                    match value.strip_prefix('+').unwrap_or(value).parse::<i32>() {
938                        Ok(n) => Shift::By(n),
939                        Err(_) => return bad(),
940                    }
941                };
942                if !seen.insert("shift") {
943                    return repeated();
944                }
945                shift = parsed;
946            }
947            // Every other `@key` is RESERVED (I1).
948            _ => return bad(),
949        }
950    }
951    ParsedDirective::Ok(Box::new(Directive {
952        path: raw.path,
953        section: raw.section,
954        lines,
955        rows,
956        cols,
957        table_attrs: None,
958        region,
959        code,
960        shift,
961    }))
962}
963
964/// Parse a whole string that must be exactly one directive (block form).
965fn parse_full_directive(text: &str) -> ParsedDirective {
966    match match_directive_at(text, 0) {
967        Some((end, raw)) if end == text.len() => parse_options(raw),
968        _ => ParsedDirective::NotADirective,
969    }
970}
971
972pub(crate) fn code_line_selector_at(line: &str, column: usize) -> bool {
973    let Some((start, _)) = line.char_indices().nth(column.saturating_sub(1)) else {
974        return false;
975    };
976    let Some((_, raw)) = match_directive_at(line, start) else {
977        return false;
978    };
979    matches!(parse_options(raw), ParsedDirective::Ok(d) if d.code.is_some() && d.lines.is_some())
980}
981
982/// An `include_directive` (PART 6) read by its shape only: any identifier key,
983/// value or not. PART 10 I2 decides a code include line from this spelling, so
984/// an unknown or malformed option must not unmake it.
985pub(crate) struct DirectiveShape {
986    pub(crate) end: usize,
987    pub(crate) quoted: bool,
988    pub(crate) options: Vec<(String, Option<String>)>,
989}
990
991/// The end of a quoted run opening at `from`, or `None` when it never closes on its line.
992fn quoted_run_end(bytes: &[u8], from: usize) -> Option<usize> {
993    let quote = bytes[from];
994    let mut i = from + 1;
995    while i < bytes.len() {
996        match bytes[i] {
997            b'\n' => return None,
998            b'\\' => i += 1,
999            c if c == quote => return Some(i),
1000            _ => {}
1001        }
1002        i += 1;
1003    }
1004    None
1005}
1006
1007fn ident_end(bytes: &[u8], from: usize, digit_start: bool) -> Option<usize> {
1008    let first = *bytes.get(from)?;
1009    if !(first.is_ascii_alphabetic() || first == b'_' || digit_start && first.is_ascii_digit()) {
1010        return None;
1011    }
1012    let mut i = from + 1;
1013    while bytes
1014        .get(i)
1015        .is_some_and(|c| c.is_ascii_alphanumeric() || matches!(c, b'_' | b'-'))
1016    {
1017        i += 1;
1018    }
1019    Some(i)
1020}
1021
1022fn skip_spaces(bytes: &[u8], mut i: usize) -> usize {
1023    while bytes.get(i) == Some(&b' ') {
1024        i += 1;
1025    }
1026    i
1027}
1028
1029/// Read the directive opening at `open` by its shape. It is parsed slot by
1030/// slot, so a failed candidate costs only what it read; the closer it reaches
1031/// must still be the first one outside a quoted run.
1032pub(crate) fn directive_shape(text: &str, open: usize) -> Option<DirectiveShape> {
1033    let bytes = text.as_bytes();
1034    if !bytes[open..].starts_with(b"{{") {
1035        return None;
1036    }
1037    let mut i = skip_spaces(bytes, open + 2);
1038    if i == open + 2 {
1039        return None;
1040    }
1041    let quoted = bytes.get(i) == Some(&b'"');
1042    if quoted {
1043        i = quoted_run_end(bytes, i)? + 1;
1044    } else {
1045        let start = i;
1046        while bytes
1047            .get(i)
1048            .is_some_and(|c| !matches!(c, b'#' | b'@' | b'}' | b' ' | b'\n'))
1049        {
1050            i += 1;
1051        }
1052        if i == start {
1053            return None;
1054        }
1055    }
1056    let j = skip_spaces(bytes, i);
1057    if bytes.get(j) == Some(&b'#') {
1058        i = ident_end(bytes, j + 1, true)?;
1059    }
1060    let mut options = Vec::new();
1061    loop {
1062        let j = skip_spaces(bytes, i);
1063        if bytes.get(j) != Some(&b'@') {
1064            break;
1065        }
1066        let key_end = ident_end(bytes, j + 1, false)?;
1067        let key = text[j + 1..key_end].to_string();
1068        i = key_end;
1069        if bytes.get(i) != Some(&b':') {
1070            options.push((key, None));
1071            continue;
1072        }
1073        if matches!(bytes.get(i + 1), Some(b'"' | b'\'')) {
1074            let end = quoted_run_end(bytes, i + 1)?;
1075            options.push((key, Some(text[i + 2..end].to_string())));
1076            i = end + 1;
1077        } else {
1078            let start = i + 1;
1079            let mut k = start;
1080            while bytes.get(k).is_some_and(|c| {
1081                !matches!(
1082                    c,
1083                    b'}' | b'|' | b'"' | b'\'' | b'\\' | b' ' | b'\t' | b'\r' | b'\n' | b'@'
1084                )
1085            }) {
1086                k += 1;
1087            }
1088            if k == start {
1089                return None;
1090            }
1091            options.push((key, Some(text[start..k].to_string())));
1092            i = k;
1093        }
1094    }
1095    let close = skip_spaces(bytes, i);
1096    // The slot parser above already consumes the only places where quotes can
1097    // shield a closer. A quote in a bare path is an ordinary path byte.
1098    if close == i || !bytes[close..].starts_with(b"}}") {
1099        return None;
1100    }
1101    Some(DirectiveShape {
1102        end: close + 2,
1103        quoted,
1104        options,
1105    })
1106}
1107
1108/// Whether `line` holds only a directive with a `code` option; `Some(quoted)` when it does.
1109pub(crate) fn code_include_line(line: &str) -> Option<bool> {
1110    let line = line.trim_matches([' ', '\t']);
1111    if !line.starts_with("{{") {
1112        return None;
1113    }
1114    let shape = directive_shape(line, 0)?;
1115    (shape.end == line.len() && shape.options.iter().any(|(key, _)| key == "code"))
1116        .then_some(shape.quoted)
1117}
1118
1119/// PART 10 I1c: `[A-Za-z0-9_+#./-]+`, as in the spec model.
1120fn is_language_info(value: &str) -> bool {
1121    !value.is_empty()
1122        && value
1123            .bytes()
1124            .all(|b| b.is_ascii_alphanumeric() || b"_+#./-".contains(&b))
1125}
1126
1127fn is_line_range(value: &str) -> bool {
1128    let Some((start, end)) = value.split_once('-') else {
1129        return false;
1130    };
1131    let decimal = |n: &str| {
1132        n.bytes().next().is_some_and(|c| (b'1'..=b'9').contains(&c))
1133            && n.bytes().all(|c| c.is_ascii_digit())
1134    };
1135    decimal(start) && decimal(end) && (start.len(), start) <= (end.len(), end)
1136}
1137
1138/// The end of a directive at `start` that stays literal text in core inline
1139/// content: an invalid `@lines` range, a `@code:lang` that is not a
1140/// `language_info` token, or a bare `@code` on a bare path.
1141pub(crate) fn literal_inline_directive_end(text: &str, start: usize) -> Option<usize> {
1142    let shape = directive_shape(text, start)?;
1143    shape
1144        .options
1145        .iter()
1146        .any(|(key, value)| match key.as_str() {
1147            "lines" => !value.as_deref().is_some_and(is_line_range),
1148            "code" => match value.as_deref() {
1149                None => !shape.quoted,
1150                Some(lang) => !is_language_info(lang),
1151            },
1152            _ => false,
1153        })
1154        .then_some(shape.end)
1155}
1156
1157/// Byte range of the first SHAPE-well-formed directive at or after `from`.
1158///
1159/// "Shape-well-formed" is deliberately WIDER than what expansion accepts: the
1160/// run opens `{{`, closes `}}`, and carries a non-empty path token - exactly
1161/// what [`match_directive_at`] recognizes. Section existence and option
1162/// validity are NOT required here; they are expansion-time DIAGNOSTICS
1163/// (`include-section`, `include-unknown-option`), and preservation must not
1164/// depend on them.
1165///
1166/// Gating preservation on full validity was the earlier behavior and it was
1167/// actively harmful: `{{ a.crv @bogus:1 }}` is a one-character typo, but
1168/// escaping it to `\{\{ … \}\}` froze the typo into permanent literal text and
1169/// destroyed the very warning that would have told the author what to fix. A
1170/// malformed directive must stay a malformed DIRECTIVE, not become prose.
1171///
1172/// A run that is not shape-well-formed - `{{ oops` with no close, or `{{ }}`
1173/// / `{{ "" }}` with an empty or whitespace-only path - is ordinary text and
1174/// is escaped as before. The empty-path exclusion lives in
1175/// [`match_directive_at`] so the serializer and the expander share exactly one
1176/// recognizer; this scanner needs no path check of its own.
1177///
1178/// This exists for the Carve serializer (spec I1): a directive must survive
1179/// `carve fmt` verbatim, because escaping it to `\{\{ … \}\}` renders the same
1180/// today but silently deletes the include the moment a resolver is wired up.
1181/// A serializer cannot tell an authored literal `{{` from a directive - both
1182/// parse to the same text - so a literal that happens to be directive-shaped
1183/// loses its escaping. That is accepted: per I9 an author who needs a
1184/// guaranteed literal puts it in a code span or fence, where the directive is
1185/// inert by construction and this scanner is never consulted.
1186fn find_directive(text: &str, from: usize) -> Option<(usize, usize)> {
1187    let mut i = from;
1188    while let Some(offset) = text[i..].find("{{") {
1189        let start = i + offset;
1190        if let Some((end, _raw)) = match_directive_at(text, start) {
1191            return Some((start, end));
1192        }
1193        // One byte, not two: in `{{{ x }}` the directive opens on the second
1194        // brace. `{` is ASCII, so this stays on a char boundary.
1195        i = start + 1;
1196    }
1197    None
1198}
1199
1200/// One piece of a run split at well-formed directive boundaries.
1201pub(crate) enum RunPiece {
1202    /// Original nodes, to be rendered normally.
1203    Nodes(Vec<InlineNode>),
1204    /// Verbatim directive source, to be emitted unescaped.
1205    Directive(String),
1206}
1207
1208/// Split a maximal run of literal-text-shaped nodes at every well-formed
1209/// directive, or `None` when it holds none.
1210///
1211/// Recognition MUST happen at run level, exactly as [`expand_run`] does it: a
1212/// directive's own syntax overlaps constructs the core already parses, so
1213/// `{{ book.crv #intro }}` arrives as Text + Tag + Text and `@shift:2` as a
1214/// Mention (I9a). Scanning individual text nodes would miss precisely the
1215/// sectioned and optioned forms.
1216pub(crate) fn split_run_directives(run: &[InlineNode]) -> Option<Vec<RunPiece>> {
1217    let full: String = run.iter().map(run_node_text).collect();
1218    if !full.contains("{{") {
1219        return None;
1220    }
1221    let slicer = RunSlicer::new(run);
1222    let mut pieces = Vec::new();
1223    let mut at = 0usize;
1224    let mut cursor = 0usize;
1225    while let Some((start, end)) = find_directive(&full, cursor) {
1226        if slicer.match_is_escaped(start, end) {
1227            cursor = start + 1;
1228            continue;
1229        }
1230        if start > at {
1231            pieces.push(RunPiece::Nodes(slicer.slice(at, start)));
1232        }
1233        pieces.push(RunPiece::Directive(full[start..end].to_string()));
1234        at = end;
1235        cursor = end;
1236    }
1237    if pieces.is_empty() {
1238        return None;
1239    }
1240    if at < full.len() {
1241        pieces.push(RunPiece::Nodes(slicer.slice(at, full.len())));
1242    }
1243    Some(pieces)
1244}
1245
1246/// The id the renderer derives from a heading's own text.
1247fn heading_slug(children: &[InlineNode]) -> String {
1248    slugify_parse(
1249        &plain_inlines(children),
1250        crate::extension::HeadingIdOptions::default(),
1251    )
1252}
1253
1254/// Loose directive shape: one whole-paragraph `{{…}}` token, valid or not.
1255fn is_directive_shaped(text: &str) -> bool {
1256    let t = text.trim();
1257    t.starts_with("{{")
1258        && t.ends_with("}}")
1259        && t.len() >= 4
1260        && !t[2..t.len() - 2].contains('{')
1261        && !t[2..t.len() - 2].contains('}')
1262}
1263
1264// ---------------------------------------------------------------------------
1265// Expansion state
1266// ---------------------------------------------------------------------------
1267
1268#[derive(Clone)]
1269struct WarningLocation {
1270    line: usize,
1271    column: usize,
1272    file: Option<String>,
1273}
1274
1275fn warning_location(pos: Option<&crate::Pos>) -> Option<WarningLocation> {
1276    pos.map(|p| WarningLocation {
1277        line: p.start_line,
1278        column: p.start_column,
1279        file: p.file.as_ref().map(|file| file.as_str().to_string()),
1280    })
1281}
1282
1283struct State<'a> {
1284    opts: &'a IncludeOptions<'a>,
1285    /// Extensions each child is parsed with.
1286    extensions: &'a [&'a dyn CarveExtension],
1287    warnings: Vec<IncludeWarning>,
1288    included: Vec<AcceptedInclude>,
1289    reach: Vec<IncludeSite>,
1290    max_depth: usize,
1291    max_bytes: usize,
1292    used_bytes: usize,
1293    max_resolver_calls: usize,
1294    resolver_calls: usize,
1295    max_warnings: usize,
1296    suppressed_warnings: usize,
1297    /// Rules already represented in `warnings`, so a cap never hides a class.
1298    seen_rules: std::collections::BTreeSet<String>,
1299    /// Position of the directive currently being expanded.
1300    warning_position: Option<WarningLocation>,
1301    /// Whether the PARENT document carries source positions, and therefore
1302    /// whether each child is parsed with them.
1303    ///
1304    /// A child parsed without them contributes nodes with no position at all,
1305    /// and the writer orders collected definitions BY position - so merged
1306    /// footnote definitions from two files tied and fell back to the label.
1307    /// It also silently drops the source mapping section 19 asks for, since
1308    /// there is no `pos` to stamp a file onto.
1309    track_positions: bool,
1310    /// Which whole-expansion total is spent, if either.
1311    ///
1312    /// Both only ever grow, so once one is spent no later directive can
1313    /// succeed. Latching stops the pass resolving the rest of the document
1314    /// only to refuse each directive individually - the refusal is already
1315    /// decided, and resolving to reach it is what turns a megabyte of
1316    /// directives into tens of thousands of file reads (spec S6).
1317    spent: Option<&'static str>,
1318    stack: Vec<String>,
1319    depth: usize,
1320    /// Identity of the document whose content is currently being expanded.
1321    file: Option<String>,
1322    /// Every element id claimed so far (I5), mapped to the file inclusion that
1323    /// claimed it first (0 is the root document, each child inclusion takes the
1324    /// next number) and the number of elements carrying it. A duplicate is
1325    /// renamed only against ANOTHER inclusion's claim; the count lets an
1326    /// element that never reaches the output give its id back without freeing
1327    /// one a surviving element still carries.
1328    used_ids: HashMap<String, (usize, usize)>,
1329    /// The last inclusion number handed out.
1330    inclusions: usize,
1331    /// Explicit-id renames waiting for their final `-N`, in the order they were
1332    /// made. The suffix has to skip every id in the ASSEMBLED document,
1333    /// including ones later includes have not contributed yet, so the merge
1334    /// writes a placeholder and [`finish_renames`] picks the name at the end.
1335    pending_renames: Vec<PendingRename>,
1336    /// The inclusion whose content is being expanded, 0 for the root.
1337    owner: usize,
1338    /// Open reservation frames, one per include being expanded (spec I7).
1339    ///
1340    /// Every identifier claimed while a child is being processed is journalled
1341    /// into the innermost frame. The frame is committed only when the child's
1342    /// content actually merges, and released otherwise, so a REJECTED
1343    /// directive leaves the document byte-identical to one that had the
1344    /// directive written as literal text from the start. Frames nest: an outer
1345    /// rejection releases everything its successful descendants claimed.
1346    reservation_frames: Vec<Vec<String>>,
1347    /// Include targets in first-encounter order, plus an index for dedup.
1348    dependencies: Vec<IncludeDependency>,
1349    dep_index: HashMap<String, usize>,
1350    /// Footnote definitions of each document on the expansion stack; the last
1351    /// entry belongs to the document currently being expanded.
1352    footnotes: Vec<BTreeMap<String, Vec<BlockNode>>>,
1353    /// Spec I8 context level C: the level of the nearest preceding heading in
1354    /// the directive's own block container or an enclosing one, 0 when there is
1355    /// none. Containers save and restore it, so a CLOSED sibling container does
1356    /// not set context.
1357    ///
1358    /// Held in the coordinate system of the content being expanded: a child
1359    /// that will later be shifted by N sees `C - N` here, so once the shift
1360    /// lands the effective context is the parent's actual level again.
1361    context_level: i32,
1362}
1363
1364impl State<'_> {
1365    fn at<T>(&mut self, position: Option<WarningLocation>, f: impl FnOnce(&mut Self) -> T) -> T {
1366        let previous = std::mem::replace(&mut self.warning_position, position);
1367        let result = f(self);
1368        self.warning_position = previous;
1369        result
1370    }
1371
1372    fn warn(&mut self, rule: &str, message: String) {
1373        let file = self.file.clone();
1374        self.warn_with_position(rule, message, file, self.warning_position.clone());
1375    }
1376
1377    fn warn_for_at(
1378        &mut self,
1379        rule: &str,
1380        message: String,
1381        file: Option<String>,
1382        pos: Option<&crate::Pos>,
1383    ) {
1384        self.warn_with_position(rule, message, file, warning_location(pos));
1385    }
1386
1387    fn warn_at(&mut self, rule: &str, message: String, pos: Option<&crate::Pos>) {
1388        let file = self.file.clone();
1389        self.warn_with_position(rule, message, file, warning_location(pos));
1390    }
1391
1392    fn warn_with_position(
1393        &mut self,
1394        rule: &str,
1395        message: String,
1396        file: Option<String>,
1397        position: Option<WarningLocation>,
1398    ) {
1399        // A rule not yet represented is always kept, so a capped report still
1400        // shows every distinct failure class; only repeats of a class already
1401        // shown are counted instead of stored.
1402        if self.warnings.len() >= self.max_warnings && self.seen_rules.contains(rule) {
1403            self.suppressed_warnings += 1;
1404            return;
1405        }
1406        self.seen_rules.insert(rule.to_string());
1407        self.warnings.push(IncludeWarning {
1408            line: position.as_ref().map_or(1, |p| p.line),
1409            column: position.as_ref().map_or(1, |p| p.column),
1410            rule: rule.to_string(),
1411            message,
1412            file: position.and_then(|p| p.file).or(file),
1413            included_by: self.reach.clone(),
1414        });
1415    }
1416
1417    /// Record an include target for host file watching. Deduplicated by id,
1418    /// first encounter fixes the order, and a later success upgrades an entry
1419    /// first seen unresolved.
1420    fn note(&mut self, id: &str, resolved: bool) {
1421        self.note_denied(id, resolved, None);
1422    }
1423
1424    fn note_denied(&mut self, id: &str, resolved: bool, denial: Option<IncludeDenial>) {
1425        match self.dep_index.get(id) {
1426            Some(&idx) => {
1427                if resolved {
1428                    self.dependencies[idx].resolved = true;
1429                    self.dependencies[idx].denial = None;
1430                } else if self.dependencies[idx].denial.is_none() {
1431                    self.dependencies[idx].denial = denial;
1432                }
1433            }
1434            None => {
1435                self.dep_index
1436                    .insert(id.to_string(), self.dependencies.len());
1437                self.dependencies.push(IncludeDependency {
1438                    id: id.to_string(),
1439                    resolved,
1440                    denial,
1441                });
1442            }
1443        }
1444    }
1445
1446    /// Claim an element id for inclusion `owner`, journalling it into the
1447    /// innermost open reservation frame. Returns `true` when the id was free.
1448    ///
1449    /// Claims made outside any frame - the root document's own ids - are
1450    /// permanent and never journalled.
1451    fn reserve_id(&mut self, id: &str, owner: usize) -> bool {
1452        let entry = self.used_ids.entry(id.to_string()).or_insert((owner, 0));
1453        entry.1 += 1;
1454        if let Some(frame) = self.reservation_frames.last_mut() {
1455            frame.push(id.to_string());
1456        }
1457        entry.1 == 1
1458    }
1459
1460    /// The inclusion that claimed `id` first, if any did.
1461    fn id_owner(&self, id: &str) -> Option<usize> {
1462        self.used_ids.get(id).map(|&(owner, _)| owner)
1463    }
1464
1465    /// Rename one occurrence of `id` to a placeholder that [`finish_renames`]
1466    /// replaces, claim it for `owner` and warn.
1467    fn defer_rename(&mut self, id: &str, owner: usize) -> String {
1468        let placeholder = format!("{RENAME_MARK}{}{RENAME_MARK}", self.pending_renames.len());
1469        self.reserve_id(&placeholder, owner);
1470        let before = self.warnings.len();
1471        self.warn_with_position(
1472            "include-id-rename",
1473            format!("Id \"{id}\" was renamed to \"{placeholder}\"."),
1474            self.file.clone(),
1475            None,
1476        );
1477        self.pending_renames.push(PendingRename {
1478            placeholder: placeholder.clone(),
1479            base: id.to_string(),
1480            warning: (self.warnings.len() > before).then_some(before),
1481        });
1482        placeholder
1483    }
1484
1485    /// Give back one element's claim on `id`, for an element that never
1486    /// reaches the output.
1487    fn release_id(&mut self, id: &str) {
1488        if let Some(entry) = self.used_ids.get_mut(id) {
1489            entry.1 = entry.1.saturating_sub(1);
1490            if entry.1 == 0 {
1491                self.used_ids.remove(id);
1492            }
1493        }
1494    }
1495
1496    fn open_reservations(&mut self) {
1497        self.reservation_frames.push(Vec::new());
1498    }
1499
1500    /// Keep the frame's claims by folding them into the enclosing frame, so an
1501    /// outer rejection can still release them.
1502    fn commit_reservations(&mut self) {
1503        let frame = self
1504            .reservation_frames
1505            .pop()
1506            .expect("reservation frames are balanced");
1507        if let Some(outer) = self.reservation_frames.last_mut() {
1508            outer.extend(frame);
1509        }
1510    }
1511
1512    /// Release every identifier the frame claimed: the child's content is not
1513    /// being merged, so nothing it named may influence the parent (I7).
1514    fn rollback_reservations(&mut self) {
1515        let frame = self
1516            .reservation_frames
1517            .pop()
1518            .expect("reservation frames are balanced");
1519        for id in frame {
1520            self.release_id(&id);
1521        }
1522    }
1523}
1524
1525mod regions;
1526pub(crate) use regions::is_table_record_marker;
1527use regions::{content_line_count, region_range, slice_lines, source_lines};
1528pub use regions::{select_include_region, IncludeRegion, IncludeRegionError, IncludeRegionMode};
1529struct ResolvedChild {
1530    source: String,
1531    id: String,
1532    position_base: Option<(String, usize)>,
1533    selected_range: Option<(usize, usize)>,
1534}
1535
1536fn resolve_child(d: &Directive, state: &mut State<'_>) -> Option<ResolvedChild> {
1537    let resolver = state.opts.resolver?;
1538    // I1: a directive selects by at most one mechanism.
1539    if d.rows.is_none()
1540        && (usize::from(d.section.is_some())
1541            + usize::from(d.lines.is_some())
1542            + usize::from(d.region.is_some())
1543            > 1
1544            || (d.code.is_some() && d.section.is_some()))
1545    {
1546        state.warn(
1547            "include-selection-conflict",
1548            format!("Include \"{}\" cannot combine selectors.", d.path),
1549        );
1550        return None;
1551    }
1552    if state.depth >= state.max_depth {
1553        // Never handed to the resolver, but still a target the host may want
1554        // to watch, so it is reported as unresolved rather than dropped.
1555        state.note(&d.path, false);
1556        state.warn(
1557            "include-depth",
1558            format!(
1559                "Include depth limit of {} exceeded for \"{}\".",
1560                state.max_depth, d.path
1561            ),
1562        );
1563        return None;
1564    }
1565
1566    // A whole-expansion total is already spent, so this directive cannot expand
1567    // whatever it resolves to. Refuse it WITHOUT resolving: the target is never
1568    // read, and it is reported unresolved because it genuinely was not.
1569    if let Some(rule) = state.spent {
1570        state.note(&d.path, false);
1571        state.warn(rule, spent_message(rule, &d.path));
1572        return None;
1573    }
1574    if state.resolver_calls >= state.max_resolver_calls {
1575        state.spent = Some("include-call-limit");
1576        state.note(&d.path, false);
1577        state.warn(
1578            "include-call-limit",
1579            spent_message("include-call-limit", &d.path),
1580        );
1581        return None;
1582    }
1583    state.resolver_calls += 1;
1584
1585    let ctx = IncludeContext {
1586        source_path: state.opts.source_path.as_deref(),
1587        stack: &state.stack,
1588        depth: state.depth,
1589    };
1590    let resolved = match resolver.resolve(&d.path, &ctx) {
1591        Ok(resolved) => resolved,
1592        Err(denial) => {
1593            // I11: a resolver that can say where the target would be reports
1594            // that path; anything else keeps the directive's spelling.
1595            let id = resolver
1596                .unresolved_id(&d.path, &ctx)
1597                .unwrap_or_else(|| d.path.clone());
1598            if denial == IncludeDenial::NonText {
1599                state.note(&id, false);
1600                state.warn(
1601                    "include-non-text",
1602                    format!("Include \"{}\" did not resolve to text.", d.path),
1603                );
1604            } else {
1605                state.note_denied(&id, false, Some(denial));
1606                state.warn(
1607                    "include-unresolved",
1608                    format!("Include \"{}\" could not be resolved.", d.path),
1609                );
1610            }
1611            return None;
1612        }
1613    };
1614    let id = resolved.id.unwrap_or_else(|| d.path.clone());
1615    let source = resolved.source;
1616    // I7: binary content warns and stays literal.
1617    if source.contains('\0') {
1618        state.note(&id, false);
1619        state.warn(
1620            "include-non-text",
1621            format!("Include \"{}\" did not resolve to text.", d.path),
1622        );
1623        return None;
1624    }
1625    state.note(&id, true);
1626    if d.rows.is_some() && (d.section.is_some() || d.lines.is_some() || d.region.is_some()) {
1627        state.warn(
1628            "include-selection-conflict",
1629            format!("Include \"{}\" cannot combine selectors.", d.path),
1630        );
1631        return None;
1632    }
1633    if (d.rows.is_some() || d.cols.is_some())
1634        && (!d
1635            .table_attrs
1636            .as_ref()
1637            .is_some_and(|a| a.key_values.get("table").is_some_and(String::is_empty))
1638            || !matches!(
1639                d.code.as_ref().and_then(|c| c.as_deref()),
1640                Some("csv" | "tsv")
1641            ))
1642    {
1643        state.warn(
1644            if d.rows.is_some() {
1645                "include-rows-invalid"
1646            } else {
1647                "include-cols-invalid"
1648            },
1649            "Include table options require a marked CSV or TSV block.".into(),
1650        );
1651        return None;
1652    }
1653    // The cycle guard compares canonical ids AFTER resolution, so a resolver
1654    // that supplies ids catches "b.crv" vs "./b.crv" spellings of one file.
1655    if state.stack.iter().any(|e| e == &id) {
1656        state.warn(
1657            "include-cycle",
1658            format!("Include cycle detected for \"{}\".", d.path),
1659        );
1660        return None;
1661    }
1662    // CHARGED BEFORE THE COMPARISON. By this line the resolver has already run
1663    // and `source` is in hand, so the read has happened whatever the budget
1664    // says. PART 9 section 19 names and accepts that: the budget bounds the
1665    // expanded OUTPUT, not the WORK, "because a target is resolved before its
1666    // size is known". Charging only what expansion ADMITS would make the
1667    // counter unable to tell "read nothing" from "read a file and refused it".
1668    // Refusing before reading is ruled out by the same clause; what bounds the
1669    // I/O is the separate resolver-call bound.
1670    state.used_bytes += source.len();
1671    if state.used_bytes > state.max_bytes {
1672        state.spent = Some("include-budget");
1673        state.warn("include-budget", spent_message("include-budget", &d.path));
1674        return None;
1675    }
1676    let range = if let Some(region) = &d.region {
1677        match region_range(&source, region, d.code.is_some()) {
1678            Some(range) => Some(range),
1679            None => {
1680                state.warn(
1681                    "include-region-invalid",
1682                    format!(
1683                        "Include \"{}\" has an invalid or missing region \"{}\".",
1684                        d.path, region
1685                    ),
1686                );
1687                return None;
1688            }
1689        }
1690    } else {
1691        if d.lines
1692            .is_some_and(|(start, _)| start > content_line_count(&source))
1693        {
1694            state.warn(
1695                "include-lines-out-of-range",
1696                format!(
1697                    "Include line range for \"{}\" starts past end of file.",
1698                    d.path
1699                ),
1700            );
1701            return None;
1702        }
1703        d.lines
1704    };
1705    let (selected, position_base) = match range {
1706        Some(range) => (
1707            slice_lines(&source, range),
1708            Some((source, range.0.saturating_sub(1))),
1709        ),
1710        None => (source, None),
1711    };
1712    Some(ResolvedChild {
1713        source: selected,
1714        id,
1715        position_base,
1716        selected_range: range,
1717    })
1718}
1719
1720fn reject_invalid_lines(path: &str, state: &mut State<'_>) {
1721    // Resolve the named file through the normal bounded path so a malformed
1722    // selector still has the dependency required by I11.
1723    let directive = Directive {
1724        path: path.to_string(),
1725        section: None,
1726        lines: None,
1727        rows: None,
1728        cols: None,
1729        table_attrs: None,
1730        region: None,
1731        code: None,
1732        shift: Shift::By(0),
1733    };
1734    let _ = resolve_child(&directive, state);
1735    state.warn(
1736        "include-lines-invalid",
1737        format!("Include line range for \"{path}\" is invalid."),
1738    );
1739}
1740
1741fn reject_invalid_table(path: &str, rule: &str, state: &mut State<'_>) {
1742    let directive = Directive {
1743        path: path.to_string(),
1744        section: None,
1745        lines: None,
1746        rows: None,
1747        cols: None,
1748        table_attrs: None,
1749        region: None,
1750        code: None,
1751        shift: Shift::By(0),
1752    };
1753    let _ = resolve_child(&directive, state);
1754    state.warn(
1755        rule,
1756        format!("Include table option for \"{path}\" is invalid."),
1757    );
1758}
1759
1760fn heading_id(h: &Heading) -> String {
1761    h.attrs
1762        .as_ref()
1763        .and_then(|a| a.id.clone())
1764        .unwrap_or_else(|| {
1765            slugify_parse(
1766                &plain_inlines(&h.children),
1767                crate::extension::HeadingIdOptions::default(),
1768            )
1769        })
1770}
1771
1772/// What `#name` selects from a child's own parse (I1a), or `None`.
1773///
1774/// Step 1: the first heading, at any depth, whose id matches: it and the blocks
1775/// after it in the same container, up to the next same-or-higher-level heading.
1776/// Step 2: otherwise the first other block carrying the explicit id. Document
1777/// order, a container before the blocks inside it. Footnote bodies are never
1778/// searched; they are not in `children`.
1779fn select_fragment(children: &[BlockNode], section: &str) -> Option<Vec<BlockNode>> {
1780    let heading = first_block(
1781        children,
1782        &|b| matches!(b, BlockNode::Heading(h) if heading_id(h) == section),
1783    );
1784    if let Some((seq, start)) = heading {
1785        let BlockNode::Heading(head) = &seq[start] else {
1786            return None;
1787        };
1788        let level = head.level;
1789        let end = seq[start + 1..]
1790            .iter()
1791            .position(|b| matches!(b, BlockNode::Heading(h) if h.level <= level))
1792            .map_or(seq.len(), |at| start + 1 + at);
1793        return Some(seq[start..end].to_vec());
1794    }
1795    let (seq, at) = first_block(children, &|b| {
1796        selectable_block_attrs(b)
1797            .and_then(|a| a.id.as_deref())
1798            .is_some_and(|id| id == section)
1799    })?;
1800    Some(vec![seq[at].clone()])
1801}
1802
1803/// The first block in document order that passes `test`, as its sequence and
1804/// index.
1805fn first_block<'a>(
1806    seq: &'a [BlockNode],
1807    test: &impl Fn(&BlockNode) -> bool,
1808) -> Option<(&'a [BlockNode], usize)> {
1809    for (i, block) in seq.iter().enumerate() {
1810        if test(block) {
1811            return Some((seq, i));
1812        }
1813        for inner in block_sequences(block) {
1814            if let Some(hit) = first_block(inner, test) {
1815                return Some(hit);
1816            }
1817        }
1818    }
1819    None
1820}
1821
1822/// The block sequences directly inside `block`: container bodies, list items,
1823/// definitions and block table cells. A line block holds lines, not blocks.
1824fn block_sequences(block: &BlockNode) -> Vec<&[BlockNode]> {
1825    fn cells(t: &crate::ast::Table) -> Vec<&[BlockNode]> {
1826        t.rows
1827            .iter()
1828            .flat_map(|r| &r.cells)
1829            .filter_map(|c| c.blocks.as_deref())
1830            .collect()
1831    }
1832    match block {
1833        BlockNode::BlockQuote(b) => vec![&b.children],
1834        BlockNode::Admonition(a) => vec![&a.children],
1835        BlockNode::Directive(d) => vec![&d.children],
1836        BlockNode::Div(d) => vec![&d.children],
1837        BlockNode::Section(d) => vec![&d.children],
1838        BlockNode::FigureGroup(g) => vec![&g.children],
1839        BlockNode::ExtensionCarrier(e) => vec![&e.children],
1840        BlockNode::BlockExtension(e) => vec![e.fallback_slice()],
1841        BlockNode::List(l) => l.items.iter().map(|i| i.children.as_slice()).collect(),
1842        BlockNode::DefinitionList(d) => d
1843            .items
1844            .iter()
1845            .flat_map(|i| &i.definitions)
1846            .map(|def| def.children.as_slice())
1847            .collect(),
1848        BlockNode::Table(t) => cells(t),
1849        BlockNode::Figure(f) => match &*f.target {
1850            FigureTarget::BlockQuote(b) => vec![&b.children],
1851            FigureTarget::Table(t) => cells(t),
1852            _ => Vec::new(),
1853        },
1854        _ => Vec::new(),
1855    }
1856}
1857
1858/// The attributes of a block step 2 may select. Definitions that render
1859/// nothing (reference, abbreviation and citation definitions, comments) and
1860/// headings (step 1's business) are never selected by a block id.
1861fn selectable_block_attrs(block: &BlockNode) -> Option<&Attrs> {
1862    match block {
1863        BlockNode::Paragraph(b) => b.attrs.as_ref(),
1864        BlockNode::CodeBlock(b) => b.attrs.as_ref(),
1865        BlockNode::List(b) => b.attrs.as_ref(),
1866        BlockNode::BlockQuote(b) => b.attrs.as_ref(),
1867        BlockNode::Table(b) => b.attrs.as_ref(),
1868        BlockNode::Admonition(b) => b.attrs.as_ref(),
1869        BlockNode::Directive(b) => b.attrs.as_ref(),
1870        BlockNode::Div(b) => b.attrs.as_ref(),
1871        BlockNode::Section(b) => b.attrs.as_ref(),
1872        BlockNode::LineBlock(b) => b.attrs.as_ref(),
1873        BlockNode::DefinitionList(b) => b.attrs.as_ref(),
1874        BlockNode::Figure(b) => b.attrs.as_ref(),
1875        BlockNode::FigureGroup(b) => b.attrs.as_ref(),
1876        BlockNode::BlockExtension(b) => b.attrs.as_ref(),
1877        BlockNode::ExtensionCarrier(b) => b.attrs.as_ref(),
1878        BlockNode::BlockImage(b) => b.attrs.as_ref(),
1879        BlockNode::ThematicBreak(b) => b.attrs.as_ref(),
1880        BlockNode::RawBlock(b) => b.attrs.as_ref(),
1881        BlockNode::Heading(_)
1882        | BlockNode::AbbreviationDef(_)
1883        | BlockNode::LinkReferenceDefinition(_)
1884        | BlockNode::CitationDefinition(_)
1885        | BlockNode::Comment(_) => None,
1886    }
1887}
1888
1889fn shift_blocks(blocks: &mut [BlockNode], shift: i32, state: &mut State<'_>) {
1890    if shift == 0 {
1891        return;
1892    }
1893    walk_blocks_mut(blocks, &mut |block| {
1894        if let BlockNode::Heading(h) = block {
1895            let shifted = h.level as i32 + shift;
1896            let clamped = shifted.clamp(1, 6);
1897            if clamped != shifted {
1898                // The heading is KEPT, never dropped (I8).
1899                state.warn_at(
1900                    "include-heading-clamp",
1901                    format!("Included heading level {shifted} was clamped to {clamped}."),
1902                    h.pos.as_ref(),
1903                );
1904            }
1905            h.level = clamped as u8;
1906        }
1907    });
1908}
1909
1910/// Spec I8 `@shift:auto`: N = (C + 1) - T, where C is the context level at the
1911/// include site and T the MINIMUM heading level in the resolved content.
1912///
1913/// The minimum rather than the first heading's level, so the child's internal
1914/// relative structure survives. Content with no headings is a no-op (N = 0)
1915/// and warns about nothing, which also covers inline includes.
1916///
1917/// Called AFTER the child's own includes are expanded, so headings a child
1918/// contributes only by including another file still count.
1919fn auto_shift(children: &[BlockNode], context_level: i32) -> i32 {
1920    let mut top: Option<u8> = None;
1921    walk_blocks(children, &mut |block| {
1922        if let BlockNode::Heading(h) = block {
1923            // `Option::is_none_or` is newer than this crate's MSRV (1.75).
1924            if top.map_or(true, |t| h.level < t) {
1925                top = Some(h.level);
1926            }
1927        }
1928    });
1929    match top {
1930        None => 0,
1931        Some(t) => context_level + 1 - t as i32,
1932    }
1933}
1934
1935/// Brackets a rename placeholder. A private-use character cannot come out of an
1936/// `{#id}` the author wrote, so a placeholder never meets a real id.
1937const RENAME_MARK: char = '\u{E000}';
1938
1939struct PendingRename {
1940    placeholder: String,
1941    base: String,
1942    /// Index of the warning that reports it, unless the warning cap dropped it.
1943    warning: Option<usize>,
1944}
1945
1946/// Give every deferred rename its final name (I5): the least `-N`, N >= 2, that
1947/// no id anywhere in the assembled document uses and no earlier rename took.
1948/// A renamed id that never reached the output (its include was rejected, or an
1949/// inline include spliced its paragraph away) renamed nothing: its warning is
1950/// dropped and references to it point at the original id again.
1951fn finish_renames(doc: &mut Document, state: &mut State<'_>) {
1952    if state.pending_renames.is_empty() {
1953        return;
1954    }
1955    let mut present: HashSet<String> = HashSet::new();
1956    let mut gather = |attrs: &mut Option<Attrs>, _: Option<&[InlineNode]>| {
1957        if let Some(id) = attrs.as_ref().and_then(|a| a.id.as_deref()) {
1958            present.insert(id.to_string());
1959        }
1960    };
1961    for_each_id_site(&mut doc.children, &mut gather);
1962    for body in doc.footnote_defs.values_mut() {
1963        for_each_id_site(body, &mut gather);
1964    }
1965    let mut taken: HashSet<String> = present
1966        .iter()
1967        .filter(|id| !id.contains(RENAME_MARK))
1968        .cloned()
1969        .collect();
1970    let mut cursor: HashMap<&str, u32> = HashMap::new();
1971    let mut rename: HashMap<String, String> = HashMap::new();
1972    let mut unsaid: Vec<usize> = Vec::new();
1973    let pending = std::mem::take(&mut state.pending_renames);
1974    for p in &pending {
1975        if !present.contains(&p.placeholder) {
1976            unsaid.extend(p.warning);
1977            rename.insert(p.placeholder.clone(), p.base.clone());
1978            continue;
1979        }
1980        // The claim it collided with did not survive (a block include
1981        // directive's own paragraph, for one): nothing to rename around.
1982        if !taken.contains(&p.base) {
1983            unsaid.extend(p.warning);
1984            taken.insert(p.base.clone());
1985            rename.insert(p.placeholder.clone(), p.base.clone());
1986            continue;
1987        }
1988        let from = cursor.get(p.base.as_str()).copied().unwrap_or(2);
1989        let (name, n) = next_free_from(&p.base, from, |c| taken.contains(c));
1990        cursor.insert(&p.base, n + 1);
1991        taken.insert(name.clone());
1992        if let Some(w) = p.warning.and_then(|i| state.warnings.get_mut(i)) {
1993            w.message = w.message.replace(&p.placeholder, &name);
1994        }
1995        rename.insert(p.placeholder.clone(), name);
1996    }
1997    for i in unsaid.into_iter().rev() {
1998        state.warnings.remove(i);
1999    }
2000    let mut apply = |attrs: &mut Option<Attrs>, _: Option<&[InlineNode]>| {
2001        if let Some(a) = attrs.as_mut() {
2002            if let Some(new) = a.id.as_ref().and_then(|id| rename.get(id)) {
2003                a.id = Some(new.clone());
2004            }
2005        }
2006    };
2007    for_each_id_site(&mut doc.children, &mut apply);
2008    for body in doc.footnote_defs.values_mut() {
2009        for_each_id_site(body, &mut apply);
2010    }
2011    let mut follow = FollowRename { rename: &rename };
2012    follow.blocks(&mut doc.children);
2013    for body in doc.footnote_defs.values_mut() {
2014        follow.blocks(body);
2015    }
2016}
2017
2018/// Merge-time collision pass for element ids (spec I5), run on one child
2019/// inclusion before its own includes expand.
2020///
2021/// An explicit id on any element is renamed when another inclusion (the parent,
2022/// an earlier include, an earlier inclusion of the same file) already claimed
2023/// it; every further copy of that id in this file is renamed too, each to its
2024/// own name. A duplicate the file holds on its own is left alone. Names compare
2025/// exactly, as HTML ids do. The final `-N` is picked by [`finish_renames`]. An
2026/// automatic heading id is re-derived from its slug against the assembled
2027/// document, silently (rule 4). References in this inclusion to a renamed id
2028/// follow its first renamed copy.
2029fn rename_child_ids(
2030    children: &mut [BlockNode],
2031    footnote_bodies: &mut BTreeMap<String, Vec<BlockNode>>,
2032    state: &mut State<'_>,
2033) -> usize {
2034    state.inclusions += 1;
2035    let me = state.inclusions;
2036    // An automatic id never lands on a name this file writes further down.
2037    let own = crate::document_ids::authored_ids(
2038        std::iter::once(&*children).chain(footnote_bodies.values().map(Vec::as_slice)),
2039        None,
2040    );
2041    // The last suffix handed out per slug, so a run of equal headings does not
2042    // restart its search at -2 each time.
2043    let mut cursor: HashMap<String, u32> = HashMap::new();
2044    let mut rename: HashMap<String, String> = HashMap::new();
2045    let mut renamed_explicit: HashSet<String> = HashSet::new();
2046    let mut visit = |attrs: &mut Option<Attrs>, heading: Option<&[InlineNode]>| {
2047        let Some(a) = attrs.as_mut() else { return };
2048        let Some(id) = a.id.clone() else { return };
2049        // A generated heading id carries no `order` slot; an authored one does.
2050        let generated = heading.filter(|_| !a.order.contains(&AttrSlot::Id));
2051        let new = if let Some(text) = generated {
2052            // The child's own suffix (`Overview-2` from its second `# Overview`)
2053            // is dropped, so the assembled document numbers it afresh.
2054            let base = heading_slug(text);
2055            let taken = |c: &str| state.used_ids.contains_key(c) || own.contains(c);
2056            let assigned = if taken(&base) {
2057                let from = cursor.get(&base).copied().unwrap_or(2);
2058                let (name, n) = next_free_from(&base, from, taken);
2059                cursor.insert(base, n + 1);
2060                name
2061            } else {
2062                base
2063            };
2064            state.reserve_id(&assigned, me);
2065            assigned
2066        } else if renamed_explicit.contains(&id) {
2067            state.defer_rename(&id, me)
2068        } else {
2069            match state.id_owner(&id) {
2070                Some(owner) if owner != me => {
2071                    renamed_explicit.insert(id.clone());
2072                    state.defer_rename(&id, me)
2073                }
2074                _ => {
2075                    state.reserve_id(&id, me);
2076                    id.clone()
2077                }
2078            }
2079        };
2080        if new != id {
2081            a.id = Some(new.clone());
2082            rename.entry(id).or_insert(new);
2083        }
2084    };
2085    for_each_id_site(children, &mut visit);
2086    for body in footnote_bodies.values_mut() {
2087        for_each_id_site(body, &mut visit);
2088    }
2089    if !rename.is_empty() {
2090        let mut follow = FollowRename { rename: &rename };
2091        for block in children.iter_mut() {
2092            follow.block(block);
2093        }
2094        for body in footnote_bodies.values_mut() {
2095            follow.blocks(body);
2096        }
2097    }
2098    me
2099}
2100
2101/// Rewrites this inclusion's references to a renamed id: a `</#id>`
2102/// cross-reference and a link or image destination that is exactly `#id`,
2103/// inline or through the file's own reference definition (I5).
2104struct FollowRename<'a> {
2105    rename: &'a HashMap<String, String>,
2106}
2107
2108impl FollowRename<'_> {
2109    /// The shared walker treats a citation entry as a leaf; its text holds
2110    /// links all the same.
2111    fn block(&mut self, block: &mut BlockNode) {
2112        if let BlockNode::CitationDefinition(d) = block {
2113            self.inlines(&mut d.children);
2114        } else {
2115            crate::include_walk::visit_block_children(block, self);
2116        }
2117    }
2118
2119    /// Whether `href` was rewritten.
2120    fn follow(&self, href: &mut String) -> bool {
2121        let Some(new) = href.strip_prefix('#').and_then(|id| self.rename.get(id)) else {
2122            return false;
2123        };
2124        *href = format!("#{new}");
2125        true
2126    }
2127}
2128
2129impl SubtreeVisitor for FollowRename<'_> {
2130    fn blocks(&mut self, blocks: &mut Vec<BlockNode>) {
2131        for block in blocks.iter_mut() {
2132            self.block(block);
2133        }
2134    }
2135
2136    fn inlines(&mut self, inlines: &mut Vec<InlineNode>) {
2137        for node in inlines.iter_mut() {
2138            match node {
2139                InlineNode::Link(l) => {
2140                    // The writer spells a reference link from its label, which
2141                    // still names the old destination: write it inline instead.
2142                    if self.follow(&mut l.href) {
2143                        l.ref_label = None;
2144                        l.raw_ref = None;
2145                    }
2146                }
2147                // A parsed `</#id>` that resolved is already a Link. One that
2148                // did not resolve in its own file is literal text, left as is.
2149                InlineNode::CrossRef(c) => {
2150                    if let Some(href) = c.href.as_mut() {
2151                        self.follow(href);
2152                        if let Some(new) = self.rename.get(&c.target) {
2153                            c.target = new.clone();
2154                        }
2155                    }
2156                }
2157                _ => {}
2158            }
2159            crate::include_walk::visit_inline_children(node, self);
2160        }
2161    }
2162
2163    fn image(&mut self, image: &mut Image) {
2164        if self.follow(&mut image.src) {
2165            image.ref_label = None;
2166            image.raw_ref = None;
2167        }
2168    }
2169}
2170
2171/// Call `f` on every attribute slot that can carry an element id, in document
2172/// order, with the heading's text when the slot is a heading's. Mirrors the
2173/// renderer's id seeding (`document_ids`).
2174fn for_each_id_site(
2175    blocks: &mut [BlockNode],
2176    f: &mut impl FnMut(&mut Option<Attrs>, Option<&[InlineNode]>),
2177) {
2178    for block in blocks {
2179        match block {
2180            BlockNode::Heading(h) => {
2181                f(&mut h.attrs, Some(&h.children));
2182                id_sites_in_inlines(&mut h.children, f);
2183            }
2184            BlockNode::Paragraph(p) => {
2185                f(&mut p.attrs, None);
2186                id_sites_in_inlines(&mut p.children, f);
2187            }
2188            BlockNode::CitationDefinition(d) => {
2189                f(&mut d.attrs, None);
2190                id_sites_in_inlines(&mut d.children, f);
2191            }
2192            BlockNode::CodeBlock(c) => f(&mut c.attrs, None),
2193            BlockNode::List(l) => {
2194                f(&mut l.attrs, None);
2195                for item in &mut l.items {
2196                    f(&mut item.attrs, None);
2197                    for_each_id_site(&mut item.children, f);
2198                }
2199            }
2200            BlockNode::BlockQuote(b) => {
2201                f(&mut b.attrs, None);
2202                for_each_id_site(&mut b.children, f);
2203            }
2204            BlockNode::Table(t) => id_sites_in_table(t, f),
2205            BlockNode::Admonition(a) => {
2206                f(&mut a.attrs, None);
2207                if let Some(title) = &mut a.title {
2208                    id_sites_in_inlines(title, f);
2209                }
2210                for_each_id_site(&mut a.children, f);
2211            }
2212            BlockNode::Directive(d) => {
2213                f(&mut d.attrs, None);
2214                if let Some(title) = &mut d.title {
2215                    id_sites_in_inlines(title, f);
2216                }
2217                for_each_id_site(&mut d.children, f);
2218            }
2219            BlockNode::Div(d) => {
2220                f(&mut d.attrs, None);
2221                for_each_id_site(&mut d.children, f);
2222            }
2223            BlockNode::Section(d) => {
2224                f(&mut d.attrs, None);
2225                for_each_id_site(&mut d.children, f);
2226            }
2227            BlockNode::LineBlock(b) => {
2228                f(&mut b.attrs, None);
2229                for_each_id_site(&mut b.children, f);
2230            }
2231            BlockNode::DefinitionList(d) => {
2232                f(&mut d.attrs, None);
2233                for item in &mut d.items {
2234                    for term in &mut item.terms {
2235                        f(&mut term.attrs, None);
2236                        id_sites_in_inlines(&mut term.children, f);
2237                    }
2238                    for def in &mut item.definitions {
2239                        for_each_id_site(&mut def.children, f);
2240                    }
2241                }
2242            }
2243            BlockNode::Figure(fig) => {
2244                f(&mut fig.attrs, None);
2245                match &mut *fig.target {
2246                    FigureTarget::Image(i) => f(&mut i.attrs, None),
2247                    FigureTarget::CodeBlock(c) => f(&mut c.attrs, None),
2248                    FigureTarget::BlockQuote(b) => {
2249                        f(&mut b.attrs, None);
2250                        for_each_id_site(&mut b.children, f);
2251                    }
2252                    FigureTarget::Table(t) => id_sites_in_table(t, f),
2253                    FigureTarget::Paragraph(p) => {
2254                        f(&mut p.attrs, None);
2255                        id_sites_in_inlines(&mut p.children, f);
2256                    }
2257                }
2258                id_sites_in_inlines(&mut fig.caption, f);
2259            }
2260            BlockNode::FigureGroup(g) => {
2261                f(&mut g.attrs, None);
2262                for_each_id_site(&mut g.children, f);
2263                if let Some(caption) = &mut g.caption {
2264                    id_sites_in_inlines(caption, f);
2265                }
2266            }
2267            BlockNode::BlockExtension(e) => {
2268                f(&mut e.attrs, None);
2269                for_each_id_site(e.fallback_slice_mut(), f);
2270            }
2271            BlockNode::ExtensionCarrier(e) => {
2272                f(&mut e.attrs, None);
2273                for_each_id_site(&mut e.children, f);
2274            }
2275            BlockNode::BlockImage(i) => f(&mut i.attrs, None),
2276            BlockNode::ThematicBreak(t) => f(&mut t.attrs, None),
2277            BlockNode::RawBlock(r) => f(&mut r.attrs, None),
2278            BlockNode::LinkReferenceDefinition(_)
2279            | BlockNode::AbbreviationDef(_)
2280            | BlockNode::Comment(_) => {}
2281        }
2282    }
2283}
2284
2285fn id_sites_in_table(
2286    t: &mut crate::ast::Table,
2287    f: &mut impl FnMut(&mut Option<Attrs>, Option<&[InlineNode]>),
2288) {
2289    f(&mut t.attrs, None);
2290    if let Some(caption) = &mut t.caption {
2291        id_sites_in_inlines(caption, f);
2292    }
2293    for row in &mut t.rows {
2294        f(&mut row.attrs, None);
2295        for cell in &mut row.cells {
2296            f(&mut cell.attrs, None);
2297            id_sites_in_inlines(&mut cell.children, f);
2298            if let Some(blocks) = &mut cell.blocks {
2299                for_each_id_site(blocks, f);
2300            }
2301        }
2302    }
2303}
2304
2305fn id_sites_in_inlines(
2306    nodes: &mut [InlineNode],
2307    f: &mut impl FnMut(&mut Option<Attrs>, Option<&[InlineNode]>),
2308) {
2309    for node in nodes {
2310        match node {
2311            InlineNode::Emphasis(e) => {
2312                f(&mut e.attrs, None);
2313                id_sites_in_inlines(&mut e.children, f);
2314            }
2315            InlineNode::Code(c) => f(&mut c.attrs, None),
2316            InlineNode::LiteralInline(l) => f(&mut l.attrs, None),
2317            InlineNode::Link(l) => {
2318                f(&mut l.attrs, None);
2319                id_sites_in_inlines(&mut l.children, f);
2320            }
2321            InlineNode::Image(i) => f(&mut i.attrs, None),
2322            InlineNode::Span(s) => {
2323                f(&mut s.attrs, None);
2324                id_sites_in_inlines(&mut s.children, f);
2325            }
2326            InlineNode::Ruby(r) => {
2327                f(&mut r.attrs, None);
2328                for pair in &mut r.pairs {
2329                    id_sites_in_inlines(&mut pair.base, f);
2330                    id_sites_in_inlines(&mut pair.annotation, f);
2331                }
2332            }
2333            InlineNode::Math(m) => f(&mut m.attrs, None),
2334            InlineNode::AutoLink(a) => f(&mut a.attrs, None),
2335            InlineNode::Extension(e) => {
2336                f(&mut e.attrs, None);
2337                id_sites_in_inlines(&mut e.children, f);
2338            }
2339            InlineNode::Footnote(n) => {
2340                f(&mut n.attrs, None);
2341                if let Some(inline) = &mut n.inline {
2342                    id_sites_in_inlines(inline, f);
2343                }
2344            }
2345            InlineNode::CriticInsert(c) => id_sites_in_inlines(&mut c.children, f),
2346            InlineNode::CriticDelete(c) => id_sites_in_inlines(&mut c.children, f),
2347            InlineNode::CriticSubstitute(c) => {
2348                id_sites_in_inlines(&mut c.old, f);
2349                id_sites_in_inlines(&mut c.new, f);
2350            }
2351            InlineNode::CitationGroup(g) => {
2352                for item in &mut g.items {
2353                    for inlines in [&mut item.prefix, &mut item.locator, &mut item.suffix]
2354                        .into_iter()
2355                        .flatten()
2356                    {
2357                        id_sites_in_inlines(inlines, f);
2358                    }
2359                }
2360            }
2361            _ => {}
2362        }
2363    }
2364}
2365
2366/// The least `base-N` with `N >= from` that `taken` refuses, and its `N`.
2367fn next_free_from(base: &str, from: u32, taken: impl Fn(&str) -> bool) -> (String, u32) {
2368    let mut n = from;
2369    loop {
2370        let candidate = format!("{base}-{n}");
2371        if !taken(&candidate) {
2372            return (candidate, n);
2373        }
2374        n += 1;
2375    }
2376}
2377
2378fn next_free(base: &str, taken: impl Fn(&str) -> bool) -> String {
2379    next_free_from(base, 2, taken).0
2380}
2381
2382fn normalize_ref_label(label: &str) -> String {
2383    label.split_whitespace().collect::<Vec<_>>().join(" ")
2384}
2385
2386/// `child_file` is the CHILD's identity: the renamed label is the child's own,
2387/// and the merge runs after expansion has already restored the parent as the
2388/// current file, so attribution is passed in explicitly.
2389fn merge_footnotes(
2390    child_defs: BTreeMap<String, Vec<BlockNode>>,
2391    child_children: &mut [BlockNode],
2392    state: &mut State<'_>,
2393    child_file: Option<String>,
2394    reach: &[IncludeSite],
2395) {
2396    if child_defs.is_empty() {
2397        return;
2398    }
2399    let mut rename: HashMap<String, String> = HashMap::new();
2400    let mut inserted: Vec<String> = Vec::new();
2401    for (label, body) in child_defs {
2402        let target = state
2403            .footnotes
2404            .last()
2405            .expect("footnote stack is never empty during expansion");
2406        let taken = target
2407            .keys()
2408            .any(|existing| normalize_ref_label(existing) == normalize_ref_label(&label));
2409        let final_label = if taken {
2410            let existing: HashSet<String> = target.keys().cloned().collect();
2411            next_free(&label, |c| existing.contains(c))
2412        } else {
2413            label.clone()
2414        };
2415        if final_label != label {
2416            let outer_reach = std::mem::replace(&mut state.reach, reach.to_vec());
2417            state.warn_for_at(
2418                "include-footnote-rename",
2419                format!("Footnote label \"{label}\" was renamed to \"{final_label}\"."),
2420                child_file.clone(),
2421                body.first().and_then(crate::ast_json::block_pos),
2422            );
2423            state.reach = outer_reach;
2424            rename.insert(label.clone(), final_label.clone());
2425        }
2426        if let Some(target) = state.footnotes.last_mut() {
2427            inserted.push(final_label.clone());
2428            target.insert(final_label, body);
2429        }
2430    }
2431    if !rename.is_empty() {
2432        rename_in_blocks(child_children, &rename);
2433        if let Some(target) = state.footnotes.last_mut() {
2434            for label in inserted {
2435                if let Some(body) = target.get_mut(&label) {
2436                    rename_in_blocks(body, &rename);
2437                }
2438            }
2439        }
2440    }
2441}
2442
2443// ---------------------------------------------------------------------------
2444// AST walking
2445// ---------------------------------------------------------------------------
2446
2447pub(crate) fn walk_blocks(blocks: &[BlockNode], f: &mut impl FnMut(&BlockNode)) {
2448    for block in blocks {
2449        f(block);
2450        match block {
2451            BlockNode::BlockQuote(b) => walk_blocks(&b.children, f),
2452            BlockNode::Directive(d) => walk_blocks(&d.children, f),
2453            BlockNode::Div(d) => walk_blocks(&d.children, f),
2454            BlockNode::Section(d) => walk_blocks(&d.children, f),
2455            BlockNode::Admonition(a) => walk_blocks(&a.children, f),
2456            BlockNode::Table(t) => {
2457                for row in &t.rows {
2458                    for cell in &row.cells {
2459                        if let Some(blocks) = &cell.blocks {
2460                            walk_blocks(blocks, f);
2461                        }
2462                    }
2463                }
2464            }
2465            BlockNode::List(l) => {
2466                for item in &l.items {
2467                    walk_blocks(&item.children, f);
2468                }
2469            }
2470            BlockNode::DefinitionList(dl) => {
2471                for item in &dl.items {
2472                    for def in &item.definitions {
2473                        walk_blocks(&def.children, f);
2474                    }
2475                }
2476            }
2477            BlockNode::Figure(fig) => match &*fig.target {
2478                FigureTarget::BlockQuote(b) => walk_blocks(&b.children, f),
2479                FigureTarget::Table(t) => {
2480                    for row in &t.rows {
2481                        for cell in &row.cells {
2482                            if let Some(blocks) = &cell.blocks {
2483                                walk_blocks(blocks, f);
2484                            }
2485                        }
2486                    }
2487                }
2488                _ => {}
2489            },
2490            _ => {}
2491        }
2492    }
2493}
2494
2495pub(crate) fn walk_blocks_mut(blocks: &mut [BlockNode], f: &mut impl FnMut(&mut BlockNode)) {
2496    for block in blocks {
2497        f(block);
2498        match block {
2499            BlockNode::BlockQuote(b) => walk_blocks_mut(&mut b.children, f),
2500            BlockNode::Directive(d) => walk_blocks_mut(&mut d.children, f),
2501            BlockNode::Div(d) => walk_blocks_mut(&mut d.children, f),
2502            BlockNode::Section(d) => walk_blocks_mut(&mut d.children, f),
2503            BlockNode::Admonition(a) => walk_blocks_mut(&mut a.children, f),
2504            BlockNode::Table(t) => {
2505                for row in &mut t.rows {
2506                    for cell in &mut row.cells {
2507                        if let Some(blocks) = &mut cell.blocks {
2508                            walk_blocks_mut(blocks, f);
2509                        }
2510                    }
2511                }
2512            }
2513            BlockNode::List(l) => {
2514                for item in &mut l.items {
2515                    walk_blocks_mut(&mut item.children, f);
2516                }
2517            }
2518            BlockNode::DefinitionList(dl) => {
2519                for item in &mut dl.items {
2520                    for def in &mut item.definitions {
2521                        walk_blocks_mut(&mut def.children, f);
2522                    }
2523                }
2524            }
2525            BlockNode::Figure(fig) => match &mut *fig.target {
2526                FigureTarget::BlockQuote(b) => walk_blocks_mut(&mut b.children, f),
2527                FigureTarget::Table(t) => {
2528                    for row in &mut t.rows {
2529                        for cell in &mut row.cells {
2530                            if let Some(blocks) = &mut cell.blocks {
2531                                walk_blocks_mut(blocks, f);
2532                            }
2533                        }
2534                    }
2535                }
2536                _ => {}
2537            },
2538            _ => {}
2539        }
2540    }
2541}
2542
2543fn rename_inlines(nodes: &mut [InlineNode], footnotes: &HashMap<String, String>) {
2544    for node in nodes {
2545        match node {
2546            InlineNode::Footnote(f) => {
2547                if let Some(id) = &f.id {
2548                    if let Some(new) = footnotes.get(id) {
2549                        f.id = Some(new.clone());
2550                    }
2551                }
2552                if let Some(inline) = &mut f.inline {
2553                    rename_inlines(inline, footnotes);
2554                }
2555            }
2556            InlineNode::Emphasis(e) => rename_inlines(&mut e.children, footnotes),
2557            InlineNode::Link(l) => rename_inlines(&mut l.children, footnotes),
2558            InlineNode::Span(s) => rename_inlines(&mut s.children, footnotes),
2559            InlineNode::Ruby(r) => {
2560                for pair in &mut r.pairs {
2561                    rename_inlines(&mut pair.base, footnotes);
2562                    rename_inlines(&mut pair.annotation, footnotes);
2563                }
2564            }
2565            InlineNode::Extension(e) => rename_inlines(&mut e.children, footnotes),
2566            InlineNode::CriticInsert(c) => rename_inlines(&mut c.children, footnotes),
2567            InlineNode::CriticDelete(c) => rename_inlines(&mut c.children, footnotes),
2568            InlineNode::CriticSubstitute(c) => {
2569                rename_inlines(&mut c.old, footnotes);
2570                rename_inlines(&mut c.new, footnotes);
2571            }
2572            InlineNode::CitationGroup(g) => {
2573                for item in &mut g.items {
2574                    for part in [&mut item.prefix, &mut item.locator, &mut item.suffix]
2575                        .into_iter()
2576                        .flatten()
2577                    {
2578                        rename_inlines(part, footnotes);
2579                    }
2580                }
2581            }
2582            _ => {}
2583        }
2584    }
2585}
2586
2587fn rename_in_blocks(blocks: &mut [BlockNode], footnotes: &HashMap<String, String>) {
2588    walk_blocks_mut(blocks, &mut |block| match block {
2589        BlockNode::Heading(h) => rename_inlines(&mut h.children, footnotes),
2590        BlockNode::Paragraph(p) => rename_inlines(&mut p.children, footnotes),
2591        BlockNode::Table(t) => {
2592            if let Some(caption) = &mut t.caption {
2593                rename_inlines(caption, footnotes);
2594            }
2595            for row in &mut t.rows {
2596                for cell in &mut row.cells {
2597                    rename_inlines(&mut cell.children, footnotes);
2598                }
2599            }
2600        }
2601        BlockNode::Figure(fig) => {
2602            rename_inlines(&mut fig.caption, footnotes);
2603            match &mut *fig.target {
2604                FigureTarget::Paragraph(p) => rename_inlines(&mut p.children, footnotes),
2605                FigureTarget::Table(t) => {
2606                    if let Some(caption) = &mut t.caption {
2607                        rename_inlines(caption, footnotes);
2608                    }
2609                    for row in &mut t.rows {
2610                        for cell in &mut row.cells {
2611                            rename_inlines(&mut cell.children, footnotes);
2612                        }
2613                    }
2614                }
2615                _ => {}
2616            }
2617        }
2618        _ => {}
2619    });
2620}
2621
2622// ---------------------------------------------------------------------------
2623// Child expansion
2624// ---------------------------------------------------------------------------
2625
2626struct ExpandedChild {
2627    children: Vec<BlockNode>,
2628    footnotes: BTreeMap<String, Vec<BlockNode>>,
2629    file: Option<String>,
2630    report: Option<AcceptedInclude>,
2631    reach: Vec<IncludeSite>,
2632}
2633
2634/// Resolve one directive and hand the result to `merge`, which decides whether
2635/// the content can actually land at the directive's position.
2636///
2637/// This is the single entry point for expanding a directive, and it enforces
2638/// spec I7 structurally rather than case by case: a reservation frame is open
2639/// for the whole of resolution AND merging, and is released unless `merge`
2640/// returns `Some`. Any rejection - unresolvable, binary, both selections,
2641/// cycle, depth, size, or content that cannot merge here - therefore leaves no
2642/// identifier claimed, so the document is byte-identical to one where the
2643/// directive was literal text from the start. Side effects added later are
2644/// covered by construction as long as they are journalled into the frame.
2645fn with_child<T>(
2646    d: &Directive,
2647    state: &mut State<'_>,
2648    inline: bool,
2649    attrs: Option<&Attrs>,
2650    merge: impl FnOnce(ExpandedChild, &mut State<'_>) -> Option<T>,
2651) -> Option<T> {
2652    state.open_reservations();
2653    let before = state.included.len();
2654    let merged = expand_child(d, state, inline).and_then(|child| {
2655        let mut report = child.report.clone();
2656        if let (Some(AcceptedInclude::Code { block, .. }), Some(attrs)) = (&mut report, attrs) {
2657            if let BlockNode::CodeBlock(code) = block.as_mut() {
2658                code.attrs = Some(attrs.clone());
2659                if !selected_table_include(d) {
2660                    if let Some(table) = table_include(code, &[]) {
2661                        **block = BlockNode::Table(table);
2662                    }
2663                }
2664            }
2665        }
2666        merge(child, state).map(|value| (value, report))
2667    });
2668    if let Some((value, report)) = merged {
2669        if let Some(report) = report {
2670            state.included.insert(before, report);
2671        }
2672        state.commit_reservations();
2673        Some(value)
2674    } else {
2675        state.included.truncate(before);
2676        state.rollback_reservations();
2677        None
2678    }
2679}
2680
2681/// Record which file a position is measured in, for every node a resolved child
2682/// contributed (spec section 19, source mapping).
2683///
2684/// Runs AFTER the child's own includes are expanded and only where no identity
2685/// is set yet, so a grandchild keeps the file IT came from rather than being
2686/// overwritten by the file that pulled its parent in.
2687///
2688/// Without it an included span is ambiguous: a child's first paragraph and the
2689/// parent's first paragraph both report line 1, and a source-mapped host has no
2690/// way to tell them apart.
2691/// Whether this document was parsed with position tracking on, read off the
2692/// tree rather than passed in: the caller hands over a parsed document, not the
2693/// options it was parsed with.
2694fn doc_carries_positions(doc: &Document) -> bool {
2695    let mut found = false;
2696    walk_blocks(&doc.children, &mut |block| {
2697        if !found && crate::ast_json::block_pos(block).is_some() {
2698            found = true;
2699        }
2700    });
2701    found
2702}
2703
2704fn stamp_source_file(blocks: &mut [BlockNode], file: &str, reach: &[IncludeSite]) {
2705    // One identity for the whole child, handed to every node in it.
2706    let file = crate::ast::SourceFile::with_include_sites(file, reach);
2707    let mut stamp = Stamp { file: &file };
2708    for block in blocks.iter_mut() {
2709        crate::include_walk::visit_block_children(block, &mut stamp);
2710    }
2711}
2712
2713/// Restore positions parsed from an `@lines` slice to the coordinates of the
2714/// complete source. This runs before nested includes are expanded, so a parent
2715/// slice base cannot be applied to a grandchild.
2716fn shift_source_positions(
2717    blocks: &mut [BlockNode],
2718    complete_source: &str,
2719    slice: &str,
2720    line_base: usize,
2721) {
2722    let mut shift = ShiftPositions {
2723        complete_line_starts: line_start_offsets(complete_source),
2724        slice_line_starts: line_start_offsets(slice),
2725        line_base,
2726    };
2727    for block in blocks.iter_mut() {
2728        crate::include_walk::visit_block_children(block, &mut shift);
2729    }
2730}
2731
2732struct ShiftPositions {
2733    complete_line_starts: Vec<usize>,
2734    slice_line_starts: Vec<usize>,
2735    line_base: usize,
2736}
2737
2738impl crate::include_walk::SubtreeVisitor for ShiftPositions {
2739    fn blocks(&mut self, blocks: &mut Vec<BlockNode>) {
2740        for block in blocks.iter_mut() {
2741            crate::include_walk::visit_block_children(block, self);
2742        }
2743    }
2744
2745    fn inlines(&mut self, inlines: &mut Vec<InlineNode>) {
2746        for node in inlines.iter_mut() {
2747            crate::include_walk::visit_inline_children(node, self);
2748        }
2749    }
2750
2751    fn position(&mut self, pos: &mut crate::ast::Pos) {
2752        pos.start_line += self.line_base;
2753        pos.end_line += self.line_base;
2754        pos.start_offset = source_offset(
2755            &self.complete_line_starts,
2756            &self.slice_line_starts,
2757            pos.start_line,
2758            pos.start_offset,
2759            self.line_base,
2760        );
2761        pos.end_offset = source_offset(
2762            &self.complete_line_starts,
2763            &self.slice_line_starts,
2764            pos.end_line,
2765            pos.end_offset,
2766            self.line_base,
2767        );
2768    }
2769}
2770
2771fn source_offset(
2772    complete_line_starts: &[usize],
2773    slice_line_starts: &[usize],
2774    complete_line: usize,
2775    offset: usize,
2776    line_base: usize,
2777) -> usize {
2778    let slice_line = complete_line.saturating_sub(line_base);
2779    complete_line_starts
2780        .get(complete_line.saturating_sub(1))
2781        .copied()
2782        .unwrap_or(*complete_line_starts.last().unwrap())
2783        + offset
2784        - slice_line_starts
2785            .get(slice_line.saturating_sub(1))
2786            .copied()
2787            .unwrap_or(*slice_line_starts.last().unwrap())
2788}
2789
2790fn line_start_offsets(source: &str) -> Vec<usize> {
2791    let mut starts = vec![0];
2792    let mut chars = source.chars().peekable();
2793    let mut offset = 0;
2794    while let Some(ch) = chars.next() {
2795        offset += 1;
2796        if ch == '\r' {
2797            if chars.next_if_eq(&'\n').is_some() {
2798                offset += 1;
2799            }
2800        } else if ch != '\n' {
2801            continue;
2802        }
2803        starts.push(offset);
2804    }
2805    if starts.last() != Some(&offset) {
2806        starts.push(offset);
2807    }
2808    starts
2809}
2810
2811/// Codepoint offset of the given 1-based physical line in source.
2812fn line_start_offset(source: &str, line: usize) -> usize {
2813    if line <= 1 {
2814        return 0;
2815    }
2816
2817    let mut offset = 0;
2818    let mut current_line = 1;
2819    let mut chars = source.chars().peekable();
2820    while let Some(ch) = chars.next() {
2821        offset += 1;
2822        if ch == '\r' {
2823            if chars.next_if_eq(&'\n').is_some() {
2824                offset += 1;
2825            }
2826        } else if ch != '\n' {
2827            continue;
2828        }
2829        current_line += 1;
2830        if current_line == line {
2831            return offset;
2832        }
2833    }
2834
2835    offset
2836}
2837
2838struct Stamp<'a> {
2839    file: &'a crate::ast::SourceFile,
2840}
2841
2842impl crate::include_walk::SubtreeVisitor for Stamp<'_> {
2843    fn blocks(&mut self, blocks: &mut Vec<BlockNode>) {
2844        for block in blocks.iter_mut() {
2845            crate::include_walk::visit_block_children(block, self);
2846        }
2847    }
2848
2849    fn inlines(&mut self, inlines: &mut Vec<InlineNode>) {
2850        for node in inlines.iter_mut() {
2851            crate::include_walk::visit_inline_children(node, self);
2852        }
2853    }
2854
2855    fn position(&mut self, pos: &mut crate::ast::Pos) {
2856        if pos.file.is_none() {
2857            pos.file = Some(self.file.clone());
2858        }
2859    }
2860}
2861
2862fn expand_child(d: &Directive, state: &mut State<'_>, inline: bool) -> Option<ExpandedChild> {
2863    let ResolvedChild {
2864        source,
2865        id,
2866        position_base,
2867        selected_range,
2868    } = resolve_child(d, state)?;
2869    let mut reach = state.reach.clone();
2870    if let Some(pos) = &state.warning_position {
2871        reach.push(IncludeSite {
2872            file: state.file.clone().unwrap_or_default(),
2873            line: pos.line,
2874            column: pos.column,
2875        });
2876    }
2877    // I4 fragment containment: the child is PARSED as a self-contained
2878    // document, never spliced as source. A construct still open at the end of
2879    // the child closes at child EOF and can never swallow parent content.
2880    let mut child_options = crate::Options::default().with_positions(state.track_positions);
2881    for extension in state.extensions {
2882        child_options = child_options.with_extension(*extension);
2883    }
2884    let mut child = if let Some(lang) = &d.code {
2885        // I1c drops the FILE's leading U+FEFF, so only a selection from line 1 sheds it.
2886        let whole = position_base
2887            .as_ref()
2888            .map_or(source.as_str(), |(whole, _)| whole.as_str());
2889        let all_lines = source_lines(whole);
2890        let (first, last) = selected_range.unwrap_or((1, all_lines.len()));
2891        let first = first.max(1);
2892        let last = last.min(all_lines.len());
2893        let lines: Vec<&str> = if whole == "\u{feff}" {
2894            Vec::new()
2895        } else if first <= last {
2896            all_lines[first - 1..last]
2897                .iter()
2898                .enumerate()
2899                .map(|(index, line)| {
2900                    if first == 1 && index == 0 {
2901                        line.strip_prefix('\u{feff}').unwrap_or(line)
2902                    } else {
2903                        line
2904                    }
2905                })
2906                .collect()
2907        } else {
2908            Vec::new()
2909        };
2910        let content = lines.iter().map(|line| format!("{line}\n")).collect();
2911        let (start_line, end_line) = if lines.is_empty() {
2912            (first, first)
2913        } else {
2914            (first, last)
2915        };
2916        let start_offset = line_start_offset(whole, start_line)
2917            + usize::from(start_line == 1 && whole.starts_with('\u{feff}'));
2918        let end_offset = if lines.is_empty() {
2919            start_offset
2920        } else {
2921            line_start_offset(whole, end_line) + all_lines[end_line - 1].chars().count()
2922        };
2923        let start_column = 1 + usize::from(start_line == 1 && whole.starts_with('\u{feff}'));
2924        let end_column = if lines.is_empty() {
2925            start_column
2926        } else {
2927            1 + all_lines[end_line - 1].chars().count()
2928        };
2929        let pos = state.track_positions.then(|| crate::Pos {
2930            start_line,
2931            end_line,
2932            start_column,
2933            end_column,
2934            start_offset,
2935            end_offset,
2936            file: Some(crate::ast::SourceFile::with_include_sites(&id, &reach)),
2937        });
2938        let mut doc = crate::parse_with_options("", &child_options);
2939        doc.children
2940            .push(BlockNode::CodeBlock(crate::ast::CodeBlock {
2941                attrs: None,
2942                lang: lang.clone(),
2943                title: None,
2944                label: None,
2945                content,
2946                pos,
2947            }));
2948        doc
2949    } else {
2950        crate::parse_with_options(&source, &child_options)
2951    };
2952    let mut children = std::mem::take(&mut child.children);
2953    let mut promoted_table = None;
2954    if let (Some(attrs), Some(Some(lang)), [BlockNode::CodeBlock(code)]) = (
2955        d.table_attrs.as_ref(),
2956        d.code.as_ref(),
2957        children.as_mut_slice(),
2958    ) {
2959        if attrs.key_values.get("table").is_some_and(String::is_empty)
2960            && matches!(lang.as_str(), "csv" | "tsv")
2961            && (d.rows.is_some() || d.cols.is_some() || selected_range.is_some())
2962        {
2963            let whole = position_base
2964                .as_ref()
2965                .map_or(source.as_str(), |(s, _)| s.as_str());
2966            let mut full = String::new();
2967            let mut payload = Vec::new();
2968            let offsets = line_start_offsets(whole);
2969            let lines = if whole == "\u{feff}" {
2970                Vec::new()
2971            } else {
2972                source_lines(whole)
2973            };
2974            for (index, line) in lines.iter().enumerate() {
2975                let text = if index == 0 {
2976                    line.strip_prefix('\u{feff}').unwrap_or(line)
2977                } else {
2978                    line
2979                };
2980                payload.push(crate::parse::delimited_table::PayloadLine {
2981                    start: full.len(),
2982                    text: text.to_string(),
2983                    line: index + 1,
2984                    column: 1 + usize::from(index == 0 && whole.starts_with('\u{feff}')),
2985                    file_offset: Some(
2986                        offsets[index] + usize::from(index == 0 && whole.starts_with('\u{feff}')),
2987                    ),
2988                });
2989                full.push_str(text);
2990                full.push('\n');
2991            }
2992            let full_code = crate::ast::CodeBlock {
2993                attrs: Some(attrs.clone()),
2994                content: full.clone(),
2995                pos: state.track_positions.then(|| crate::Pos {
2996                    start_line: 1,
2997                    end_line: lines.len().max(1),
2998                    start_column: 1 + usize::from(whole.starts_with('\u{feff}')),
2999                    end_column: lines.last().map_or(1, |line| 1 + line.chars().count()),
3000                    start_offset: usize::from(whole.starts_with('\u{feff}')),
3001                    end_offset: whole.chars().count(),
3002                    file: Some(crate::ast::SourceFile::with_include_sites(&id, &reach)),
3003                }),
3004                ..code.clone()
3005            };
3006            let sliced_valid = selected_range.is_none()
3007                || table_include(
3008                    &crate::ast::CodeBlock {
3009                        attrs: Some(attrs.clone()),
3010                        ..code.clone()
3011                    },
3012                    &[],
3013                )
3014                .is_some();
3015            if let Some(mut table) = table_include(&full_code, &payload).filter(|table| {
3016                sliced_valid
3017                    && selected_range.map_or(true, |(first, last)| {
3018                        table.rows.iter().all(|row| {
3019                            row.pos.as_ref().map_or(true, |pos| {
3020                                !(pos.start_line < first && pos.end_line >= first
3021                                    || pos.start_line <= last && pos.end_line > last)
3022                            })
3023                        })
3024                    })
3025            }) {
3026                let headers = table
3027                    .rows
3028                    .iter()
3029                    .take_while(|row| row.cells.first().is_some_and(|cell| cell.header))
3030                    .count();
3031                if d.rows.is_some_and(|(first, _)| first > table.rows.len()) {
3032                    state.warn(
3033                        "include-rows-out-of-range",
3034                        format!(
3035                            "Include record range for \"{}\" starts past end of file.",
3036                            d.path
3037                        ),
3038                    );
3039                    return None;
3040                }
3041                let mut selected = Vec::new();
3042                for (index, mut row) in table.rows.drain(..).enumerate() {
3043                    let record = index + 1;
3044                    let keep = index < headers
3045                        || if let Some((first, last)) = d.rows {
3046                            record >= first && record <= last
3047                        } else if let Some((first, last)) = selected_range {
3048                            row.pos.as_ref().is_some_and(|pos| {
3049                                pos.start_line >= first && pos.start_line <= last
3050                            })
3051                        } else {
3052                            true
3053                        };
3054                    if keep {
3055                        if index >= headers {
3056                            for cell in &mut row.cells {
3057                                cell.header = false;
3058                            }
3059                        }
3060                        selected.push(row);
3061                    }
3062                }
3063                table.rows = selected;
3064                if let Some(cols) = &d.cols {
3065                    let indices = match cols {
3066                        IncludeColumns::Names(names) => {
3067                            if headers == 0 {
3068                                state.warn(
3069                                    "include-cols-invalid",
3070                                    "Named include columns require a header row.".into(),
3071                                );
3072                                return None;
3073                            }
3074                            let labels: Vec<String> = table.rows[0]
3075                                .cells
3076                                .iter()
3077                                .map(|cell| plain_inlines(&cell.children))
3078                                .collect();
3079                            let mut indices = Vec::new();
3080                            for name in names {
3081                                let matching: Vec<usize> = labels
3082                                    .iter()
3083                                    .enumerate()
3084                                    .filter_map(|(i, label)| (label == name).then_some(i))
3085                                    .collect();
3086                                if matching.len() != 1 {
3087                                    state.warn("include-cols-invalid", format!("Include column \"{name}\" is missing, empty, or ambiguous."));
3088                                    return None;
3089                                }
3090                                if indices.contains(&matching[0]) {
3091                                    state.warn(
3092                                        "include-cols-duplicate",
3093                                        format!("Include column \"{name}\" is duplicated."),
3094                                    );
3095                                    return None;
3096                                }
3097                                indices.push(matching[0]);
3098                            }
3099                            indices
3100                        }
3101                        IncludeColumns::Range(first, last) => {
3102                            if attrs.key_values.get("header-rows").map(String::as_str) != Some("0")
3103                                || *last > table.rows.first().map_or(0, |row| row.cells.len())
3104                            {
3105                                state.warn("include-cols-invalid", "Numeric include columns require header-rows=0 and an in-bounds range.".into());
3106                                return None;
3107                            }
3108                            (*first - 1..*last).collect()
3109                        }
3110                    };
3111                    for row in &mut table.rows {
3112                        row.cells = indices
3113                            .iter()
3114                            .filter_map(|&i| row.cells.get(i).cloned())
3115                            .collect();
3116                    }
3117                }
3118                table.row_groups = None;
3119                promoted_table = Some(table);
3120            } else if d.rows.is_some() || d.cols.is_some() {
3121                code.content = full;
3122                code.pos = full_code.pos;
3123            }
3124        }
3125    }
3126    if let Some(table) = promoted_table {
3127        children[0] = BlockNode::Table(table);
3128    }
3129    let mut footnotes = std::mem::take(&mut child.footnote_defs);
3130    if let Some((complete_source, line_base)) = position_base.as_ref().filter(|_| d.code.is_none())
3131    {
3132        shift_source_positions(&mut children, complete_source, &source, *line_base);
3133        for body in footnotes.values_mut() {
3134            shift_source_positions(body, complete_source, &source, *line_base);
3135        }
3136    }
3137    // Select BEFORE expanding: nested includes outside the wanted section must
3138    // not be resolved (no budget charge) and must not move section boundaries.
3139    if let Some(section) = &d.section {
3140        match select_fragment(&children, section) {
3141            Some(selected) => children = selected,
3142            None => {
3143                // I11: `resolved` reports whether the target's SOURCE WAS READ,
3144                // nothing more. The read succeeded here - only the section
3145                // selection failed - so the entry stays resolved and the host
3146                // keeps watching the file. Downgrading it would stop the watch
3147                // at exactly the wrong moment: adding the missing section to
3148                // the child would then never invalidate the preview.
3149                state.warn(
3150                    "include-section",
3151                    format!("Include \"{}\" has no section \"#{section}\".", d.path),
3152                );
3153                return None;
3154            }
3155        }
3156    }
3157
3158    let report = if state.opts.collect_included {
3159        if d.code.is_some() {
3160            // Lint checks the content the renderer converts, not the raw bytes.
3161            let payload = match children.first()? {
3162                BlockNode::CodeBlock(code) => code.content.clone(),
3163                _ => source.clone(),
3164            };
3165            Some(AcceptedInclude::Code {
3166                file: id.clone(),
3167                included_by: reach.clone(),
3168                block: Box::new(children.first()?.clone()),
3169                payload,
3170            })
3171        } else {
3172            let selected_lines = d.section.as_ref().and_then(|_| {
3173                let first = children
3174                    .first()
3175                    .and_then(crate::ast_json::block_pos)?
3176                    .start_line;
3177                let last = children
3178                    .last()
3179                    .and_then(crate::ast_json::block_pos)?
3180                    .end_line;
3181                Some((first, last))
3182            });
3183            Some(AcceptedInclude::Carve {
3184                file: id.clone(),
3185                included_by: reach.clone(),
3186                source: source.clone(),
3187                first_line: position_base.as_ref().map_or(0, |(_, base)| *base),
3188                selected_lines,
3189            })
3190        }
3191    } else {
3192        None
3193    };
3194    // Everything from here on operates on the child's own content, so warnings
3195    // it raises name the CHILD rather than the document that included it.
3196    let outer_file = state.file.replace(id.clone());
3197    // An inline include splices a lone paragraph's inlines and leaves its
3198    // attributes behind, so its id must not claim a name it never takes.
3199    if inline {
3200        if let [BlockNode::Paragraph(p)] = children.as_mut_slice() {
3201            p.attrs = None;
3202        }
3203    }
3204    let me = rename_child_ids(&mut children, &mut footnotes, state);
3205    let outer_owner = std::mem::replace(&mut state.owner, me);
3206
3207    let auto = d.shift == Shift::Auto;
3208    let stated = match d.shift {
3209        Shift::By(n) => n,
3210        Shift::Auto => 0,
3211    };
3212    let outer_reach = std::mem::replace(&mut state.reach, reach.clone());
3213    state.stack.push(id.clone());
3214    state.depth += 1;
3215    state.footnotes.push(footnotes);
3216    // The child is shifted only AFTER its own includes are expanded, so inside
3217    // it the inherited context is expressed in pre-shift coordinates: a stated
3218    // shift is known now and translated out, and once it lands a nested `auto`
3219    // sits where the assembled document says it should.
3220    //
3221    // `auto` is NOT translated because its offset is not known yet - it is
3222    // measured over the assembled content below, which is exactly what makes
3223    // it self-consistent.
3224    let outer_context = state.context_level;
3225    state.context_level = outer_context - stated;
3226    expand_blocks(&mut children, state);
3227    let mut footnotes = state
3228        .footnotes
3229        .pop()
3230        .expect("footnote stack push/pop are balanced");
3231    // A footnote body is its own container: no heading precedes it.
3232    state.footnotes.push(footnotes);
3233    let labels: Vec<String> = state.footnotes.last().unwrap().keys().cloned().collect();
3234    for label in labels {
3235        let mut body = state.footnotes.last_mut().unwrap().remove(&label).unwrap();
3236        state.context_level = 0;
3237        expand_blocks(&mut body, state);
3238        state.footnotes.last_mut().unwrap().insert(label, body);
3239    }
3240    footnotes = state
3241        .footnotes
3242        .pop()
3243        .expect("footnote stack push/pop are balanced");
3244    state.context_level = outer_context;
3245    state.depth -= 1;
3246    state.stack.pop();
3247    state.reach = outer_reach;
3248    state.owner = outer_owner;
3249    // A paragraph a nested include contributed is spliced the same way.
3250    let dropped = match children.as_mut_slice() {
3251        [BlockNode::Paragraph(p)] if inline => p.attrs.take().and_then(|a| a.id),
3252        _ => None,
3253    };
3254    if let Some(id) = dropped {
3255        state.release_id(&id);
3256    }
3257    // Measured after expansion so a child that only passes through to nested
3258    // includes is levelled by the headings those actually contributed.
3259    let shift = if auto {
3260        auto_shift(&children, state.context_level)
3261    } else {
3262        stated
3263    };
3264    shift_blocks(&mut children, shift, state);
3265    // After the child's own includes, so a grandchild keeps its own identity.
3266    stamp_source_file(&mut children, &id, &reach);
3267    // THE FOOTNOTE BODIES TOO. They travel beside the blocks rather than in
3268    // them, so stamping only `children` left every merged definition with no
3269    // file identity - and the writer orders collected definitions by source
3270    // position, which is meaningless across files without one. Two children
3271    // each defining a note near their own start then tied, and the order fell
3272    // back to the label.
3273    for body in footnotes.values_mut() {
3274        stamp_source_file(body, &id, &reach);
3275    }
3276    state.file = outer_file;
3277    Some(ExpandedChild {
3278        children,
3279        footnotes,
3280        file: Some(id),
3281        report,
3282        reach,
3283    })
3284}
3285
3286// ---------------------------------------------------------------------------
3287// Inline expansion (I2 inline form, I9a run recognition)
3288// ---------------------------------------------------------------------------
3289
3290pub(crate) fn is_run_node(node: &InlineNode) -> bool {
3291    matches!(
3292        node,
3293        InlineNode::Text(_)
3294            | InlineNode::Mention(_)
3295            | InlineNode::Tag(_)
3296            | InlineNode::SmartPunctuation(_)
3297            | InlineNode::EscapedText(_)
3298    )
3299}
3300
3301/// Source form of a run node. A directive's own syntax overlaps constructs the
3302/// core already parses (`#section` is TAG syntax, `@key:value` is MENTION
3303/// syntax), so recognition reassembles the run before matching (I9a).
3304fn run_node_text(node: &InlineNode) -> String {
3305    match node {
3306        InlineNode::Text(t) => t.value.clone(),
3307        InlineNode::Mention(m) => format!("@{}", m.user),
3308        InlineNode::Tag(t) => format!("#{}", t.name),
3309        // A QUOTED PATH is two more of those constructs: its delimiters arrive
3310        // as SmartPunctuation whose `value` is the quote the author typed
3311        // rather than the “ smart typography resolved it to, and an escaped
3312        // quote inside the path arrives as EscapedText without its backslash.
3313        // Reassembling from the author's spelling is what the grammar matches.
3314        InlineNode::SmartPunctuation(p) => p.value.clone(),
3315        InlineNode::EscapedText(t) => format!("\\{}", t.value),
3316        _ => String::new(),
3317    }
3318}
3319
3320/// The byte length `run_node_text` gives a node, without building the string.
3321fn run_node_len(node: &InlineNode) -> usize {
3322    match node {
3323        InlineNode::Text(t) => t.value.len(),
3324        InlineNode::Mention(m) => 1 + m.user.len(),
3325        InlineNode::Tag(t) => 1 + t.name.len(),
3326        InlineNode::SmartPunctuation(p) => p.value.len(),
3327        InlineNode::EscapedText(t) => 1 + t.value.len(),
3328        _ => 0,
3329    }
3330}
3331
3332/// A run's nodes addressed by offsets into its reassembled text.
3333///
3334/// Built ONCE per run. Slicing used to re-measure every node from the start
3335/// and clone each text node whole for every directive, so a paragraph that is
3336/// one long text node holding many directives cost the square of its length
3337/// (markup-carve/carve-rs#1618).
3338struct RunSlicer<'a> {
3339    run: &'a [InlineNode],
3340    ends: Vec<usize>,
3341}
3342
3343fn advance_line_column(mut line: usize, mut column: usize, text: &str) -> (usize, usize) {
3344    let mut chars = text.chars().peekable();
3345    while let Some(ch) = chars.next() {
3346        match ch {
3347            '\r' => {
3348                if chars.peek() == Some(&'\n') {
3349                    chars.next();
3350                }
3351                line += 1;
3352                column = 1;
3353            }
3354            '\n' => {
3355                line += 1;
3356                column = 1;
3357            }
3358            _ => column += 1,
3359        }
3360    }
3361    (line, column)
3362}
3363
3364impl<'a> RunSlicer<'a> {
3365    fn new(run: &'a [InlineNode]) -> Self {
3366        let mut end = 0usize;
3367        let ends = run
3368            .iter()
3369            .map(|node| {
3370                end += run_node_len(node);
3371                end
3372            })
3373            .collect();
3374        Self { run, ends }
3375    }
3376
3377    /// Location of a directive opener within this reassembled run. Braces
3378    /// always start in a text node, even when options span tags or mentions.
3379    fn position(&self, at: usize) -> Option<WarningLocation> {
3380        let i = self.ends.partition_point(|end| *end <= at);
3381        let InlineNode::Text(text) = self.run.get(i)? else {
3382            return None;
3383        };
3384        let pos = text.pos.as_ref()?;
3385        let node_start = if i == 0 { 0 } else { self.ends[i - 1] };
3386        let prefix = text.value.get(..at - node_start)?;
3387        let (line, column) = advance_line_column(pos.start_line, pos.start_column, prefix);
3388        Some(WarningLocation {
3389            line,
3390            column,
3391            file: pos.file.as_ref().map(|file| file.as_str().to_string()),
3392        })
3393    }
3394
3395    /// Whether a match over `[start, end)` opens or closes on a backslash
3396    /// escape. An escaped brace is literal (CARVE-P7-004), so such a match is
3397    /// no directive: `\{{ x }}` holds no `{{`.
3398    fn match_is_escaped(&self, start: usize, end: usize) -> bool {
3399        let escaped_at = |at: usize| {
3400            let i = self.ends.partition_point(|e| *e <= at);
3401            matches!(self.run.get(i), Some(InlineNode::EscapedText(_)))
3402        };
3403        escaped_at(start) || escaped_at(end - 1)
3404    }
3405
3406    /// The run nodes covering `[from, to)` of the reassembled text. Directive
3407    /// matches start with `{{` and end with `}}`, which the core always parses
3408    /// as text, so a boundary can only fall inside a text node; mention and tag
3409    /// nodes are either fully kept or fully consumed by a directive span.
3410    fn slice(&self, from: usize, to: usize) -> Vec<InlineNode> {
3411        let mut out = Vec::new();
3412        let first = self.ends.partition_point(|end| *end <= from);
3413        for (i, node) in self.run.iter().enumerate().skip(first) {
3414            let end = self.ends[i];
3415            let start = end - run_node_len(node);
3416            if start >= to {
3417                break;
3418            }
3419            let InlineNode::Text(value) = node else {
3420                out.push(node.clone());
3421                continue;
3422            };
3423            let lo = from.max(start) - start;
3424            let hi = to.min(end) - start;
3425            if lo == 0 && hi == value.value.len() {
3426                out.push(node.clone());
3427            } else if lo < hi {
3428                out.push(InlineNode::Text(crate::ast::Text {
3429                    value: value.value[lo..hi].to_string(),
3430                    pos: value.pos.clone(),
3431                }));
3432            }
3433        }
3434        out
3435    }
3436}
3437
3438fn expand_run(run: &[InlineNode], state: &mut State<'_>) -> Vec<InlineNode> {
3439    let full: String = run.iter().map(run_node_text).collect();
3440    if !full.contains("{{") {
3441        return run.to_vec();
3442    }
3443    let slicer = RunSlicer::new(run);
3444    let mut spans: Vec<(usize, usize, Vec<InlineNode>)> = Vec::new();
3445    let mut cursor = 0usize;
3446    while let Some(rel) = full[cursor..].find("{{") {
3447        let start = cursor + rel;
3448        let Some((end, raw)) = match_directive_at(&full, start) else {
3449            cursor = start + 1;
3450            continue;
3451        };
3452        if slicer.match_is_escaped(start, end) {
3453            cursor = start + 1;
3454            continue;
3455        }
3456        cursor = end;
3457        let replacement = state.at(slicer.position(start), |state| {
3458            let d = match parse_options(raw) {
3459                ParsedDirective::Ok(d) => d,
3460                ParsedDirective::InvalidLines(path) => {
3461                    reject_invalid_lines(&path, state);
3462                    return None;
3463                }
3464                ParsedDirective::InvalidTable(path, rule) => {
3465                    reject_invalid_table(&path, rule, state);
3466                    return None;
3467                }
3468                ParsedDirective::BadOption(part) => {
3469                    state.warn(
3470                        "include-unknown-option",
3471                        format!("Unknown include option \"{part}\"."),
3472                    );
3473                    return None;
3474                }
3475                ParsedDirective::DuplicateSection(part) => {
3476                    state.warn(
3477                        "include-selection-conflict",
3478                        format!("Include directive cannot name two sections: \"{part}\"."),
3479                    );
3480                    return None;
3481                }
3482                ParsedDirective::DuplicateOption(part) => {
3483                    state.warn(
3484                        "include-duplicate-option",
3485                        format!("Include option \"{part}\" repeats an option given earlier."),
3486                    );
3487                    return None;
3488                }
3489                ParsedDirective::RepeatedTableOption(path, part) => {
3490                    let directive = Directive {
3491                        path,
3492                        section: None,
3493                        lines: None,
3494                        rows: None,
3495                        cols: None,
3496                        table_attrs: None,
3497                        region: None,
3498                        code: None,
3499                        shift: Shift::By(0),
3500                    };
3501                    let _ = resolve_child(&directive, state);
3502                    state.warn(
3503                        "include-duplicate-option",
3504                        format!("Include option \"{part}\" repeats an option given earlier."),
3505                    );
3506                    return None;
3507                }
3508                ParsedDirective::NotADirective => return None,
3509            };
3510            with_child(&d, state, true, None, |expanded, state| {
3511                let mut children = expanded.children;
3512                // I2: resolved content in INLINE position must parse to inline-only
3513                // content - a single paragraph, or nothing.
3514                let inline_only = match children.len() {
3515                    0 => true,
3516                    1 => matches!(children[0], BlockNode::Paragraph(_)),
3517                    _ => false,
3518                };
3519                if !inline_only {
3520                    state.warn(
3521                        "include-block-in-inline",
3522                        format!("Inline include \"{}\" resolved to block content.", d.path),
3523                    );
3524                    return None;
3525                }
3526                // Merge BEFORE lifting the paragraph's inlines out: a footnote-label
3527                // rename rewrites the references inside `children`, so taking them
3528                // first would rename the definition while leaving the spliced
3529                // reference pointing at the label the parent kept.
3530                merge_footnotes(
3531                    expanded.footnotes,
3532                    &mut children,
3533                    state,
3534                    expanded.file,
3535                    &expanded.reach,
3536                );
3537                Some(match children.first_mut() {
3538                    Some(BlockNode::Paragraph(p)) => std::mem::take(&mut p.children),
3539                    _ => Vec::new(),
3540                })
3541            })
3542        });
3543        let Some(replacement) = replacement else {
3544            continue;
3545        };
3546        spans.push((start, end, replacement));
3547    }
3548    if spans.is_empty() {
3549        return run.to_vec();
3550    }
3551    let mut pieces: Vec<(InlineNode, bool)> = Vec::new();
3552    let mut at = 0usize;
3553    for (start, end, replacement) in spans {
3554        pieces.extend(slicer.slice(at, start).into_iter().map(|n| (n, true)));
3555        pieces.extend(replacement.into_iter().map(|n| (n, false)));
3556        at = end;
3557    }
3558    pieces.extend(slicer.slice(at, full.len()).into_iter().map(|n| (n, true)));
3559    coalesce_text(pieces)
3560}
3561
3562/// PART 12 §1a: a serialized node's children hold no two adjacent `text` nodes,
3563/// and §1a's own POSITIONS paragraph says the coalescing happens in the parsed
3564/// tree rather than in the encoder. Splicing an inline include's content into
3565/// the host's run is the one place this pass leaves a run split, so it joins it
3566/// back here (CARVE-P12-002, markup-carve/carve-rs#1647).
3567///
3568/// The merged run spans the FILE THAT HOLDS IT, from the first host piece's
3569/// start to the last host piece's end. A run with no positioned host piece
3570/// publishes none: an included piece's coordinates are measured in another
3571/// file and would select the wrong text here.
3572fn coalesce_text(pieces: Vec<(InlineNode, bool)>) -> Vec<InlineNode> {
3573    let mut out: Vec<InlineNode> = Vec::with_capacity(pieces.len());
3574    let mut run: Vec<(InlineNode, bool)> = Vec::new();
3575
3576    fn flush(run: &mut Vec<(InlineNode, bool)>, out: &mut Vec<InlineNode>) {
3577        if run.len() < 2 {
3578            out.extend(run.drain(..).map(|(node, _)| node));
3579            return;
3580        }
3581        let mut value = String::new();
3582        let mut first: Option<crate::Pos> = None;
3583        let mut last: Option<crate::Pos> = None;
3584        for (node, from_host) in run.drain(..) {
3585            let InlineNode::Text(text) = node else {
3586                continue;
3587            };
3588            value.push_str(&text.value);
3589            if !from_host {
3590                continue;
3591            }
3592            if let Some(pos) = text.pos {
3593                first.get_or_insert_with(|| pos.clone());
3594                last = Some(pos);
3595            }
3596        }
3597        let pos = match (first, last) {
3598            (Some(first), Some(last)) => Some(crate::Pos {
3599                end_line: last.end_line,
3600                end_column: last.end_column,
3601                end_offset: last.end_offset,
3602                ..first
3603            }),
3604            _ => None,
3605        };
3606        out.push(InlineNode::Text(crate::ast::Text { value, pos }));
3607    }
3608
3609    for (node, from_host) in pieces {
3610        if matches!(node, InlineNode::Text(_)) {
3611            run.push((node, from_host));
3612            continue;
3613        }
3614        flush(&mut run, &mut out);
3615        out.push(node);
3616    }
3617    flush(&mut run, &mut out);
3618    out
3619}
3620
3621fn expand_inlines(nodes: &mut Vec<InlineNode>, state: &mut State<'_>) {
3622    let mut out: Vec<InlineNode> = Vec::with_capacity(nodes.len());
3623    let mut i = 0usize;
3624    while i < nodes.len() {
3625        if is_run_node(&nodes[i]) {
3626            // A directive split across other inline structures (emphasis, a
3627            // link, a code span) stays literal by design: the run STOPS at any
3628            // node that is not literal-text-shaped (I9a), which is also what
3629            // gives code spans their verbatim protection (I9).
3630            let mut j = i;
3631            while j < nodes.len() && is_run_node(&nodes[j]) {
3632                j += 1;
3633            }
3634            out.extend(expand_run(&nodes[i..j], state));
3635            i = j;
3636            continue;
3637        }
3638        let mut node = nodes[i].clone();
3639        match &mut node {
3640            InlineNode::Emphasis(e) => expand_inlines(&mut e.children, state),
3641            InlineNode::Link(l) => expand_inlines(&mut l.children, state),
3642            InlineNode::Span(s) => expand_inlines(&mut s.children, state),
3643            InlineNode::Ruby(r) => {
3644                for pair in &mut r.pairs {
3645                    expand_inlines(&mut pair.base, state);
3646                    expand_inlines(&mut pair.annotation, state);
3647                }
3648            }
3649            InlineNode::Extension(e) => expand_inlines(&mut e.children, state),
3650            InlineNode::CriticInsert(c) => expand_inlines(&mut c.children, state),
3651            InlineNode::CriticDelete(c) => expand_inlines(&mut c.children, state),
3652            InlineNode::CriticSubstitute(c) => {
3653                expand_inlines(&mut c.old, state);
3654                expand_inlines(&mut c.new, state);
3655            }
3656            InlineNode::Footnote(f) => {
3657                if let Some(inline) = &mut f.inline {
3658                    expand_inlines(inline, state);
3659                }
3660            }
3661            InlineNode::CitationGroup(g) => {
3662                for item in &mut g.items {
3663                    for part in [&mut item.prefix, &mut item.locator, &mut item.suffix]
3664                        .into_iter()
3665                        .flatten()
3666                    {
3667                        expand_inlines(part, state);
3668                    }
3669                }
3670            }
3671            // Code, RawInline and Math are VERBATIM (I9): never traversed, so
3672            // a directive inside them is never handed to the resolver.
3673            _ => {}
3674        }
3675        out.push(node);
3676        i += 1;
3677    }
3678    *nodes = out;
3679}
3680
3681// ---------------------------------------------------------------------------
3682// Block expansion (I2 block form)
3683// ---------------------------------------------------------------------------
3684
3685/// Reassemble a paragraph's inlines into directive source, or `None` if the
3686/// paragraph holds anything that is not literal-text-shaped.
3687fn directive_source(nodes: &[InlineNode]) -> Option<String> {
3688    let mut out = String::new();
3689    for node in nodes {
3690        if !is_run_node(node) {
3691            return None;
3692        }
3693        out.push_str(&run_node_text(node));
3694    }
3695    Some(out)
3696}
3697
3698fn expand_paragraph(block: &mut Paragraph, state: &mut State<'_>) -> Option<Vec<BlockNode>> {
3699    if block
3700        .attrs
3701        .as_ref()
3702        .is_some_and(|attrs| attrs.key_values.get("table").is_some_and(String::is_empty))
3703    {
3704        if let Some(split) = block
3705            .children
3706            .iter()
3707            .position(|node| matches!(node, InlineNode::SoftBreak(_)))
3708        {
3709            if let Some(source) = directive_source(&block.children[..split]) {
3710                if matches!(parse_full_directive(&source), ParsedDirective::Ok(ref d) if d.code.is_some())
3711                {
3712                    let mut lead = block.clone();
3713                    lead.children.truncate(split);
3714                    if let Some(mut replacement) = expand_paragraph(&mut lead, state) {
3715                        if split + 1 < block.children.len() {
3716                            let mut tail = block.clone();
3717                            tail.attrs = None;
3718                            tail.children = block.children[split + 1..].to_vec();
3719                            let caption =
3720                                if let Some(InlineNode::Text(text)) = tail.children.first_mut() {
3721                                    text.value.strip_prefix("^ ").map(str::to_string)
3722                                } else {
3723                                    None
3724                                };
3725                            if let Some(first) = caption {
3726                                if let Some(InlineNode::Text(text)) = tail.children.first_mut() {
3727                                    text.value = first;
3728                                }
3729                                if let [BlockNode::Table(table)] = replacement.as_mut_slice() {
3730                                    table.caption = Some(tail.children);
3731                                } else if let [BlockNode::CodeBlock(code)] =
3732                                    replacement.as_mut_slice()
3733                                {
3734                                    replacement = vec![BlockNode::Figure(crate::ast::Figure {
3735                                        attrs: None,
3736                                        target: Box::new(FigureTarget::CodeBlock(code.clone())),
3737                                        rendered_target: None,
3738                                        caption: tail.children,
3739                                        short_caption: None,
3740                                        pos: code.pos.clone(),
3741                                    })];
3742                                } else {
3743                                    replacement.push(BlockNode::Paragraph(tail));
3744                                }
3745                            } else {
3746                                replacement.push(BlockNode::Paragraph(tail));
3747                            }
3748                        }
3749                        return Some(replacement);
3750                    }
3751                }
3752            }
3753        }
3754    }
3755    if let Some(source) = directive_source(&block.children) {
3756        let parsed = parse_full_directive(&source);
3757        let position = RunSlicer::new(&block.children).position(0);
3758        match parsed {
3759            ParsedDirective::Ok(mut d) => {
3760                d.table_attrs = block.attrs.clone();
3761                return state.at(position, |state| {
3762                    // I7: on any rejection this yields None and the original inline
3763                    // nodes stay, rendering exactly as the core does with no
3764                    // resolver - and `with_child` releases what the child claimed.
3765                    // The paragraph is replaced by what it includes, so its id is
3766                    // given back while the child claims; left literal, it keeps it.
3767                    // A code include keeps the id: it moves onto the code block.
3768                    let held = block
3769                        .attrs
3770                        .as_ref()
3771                        .and_then(|a| a.id.clone())
3772                        .filter(|id| d.code.is_none() && state.id_owner(id) == Some(state.owner));
3773                    if let Some(id) = &held {
3774                        state.release_id(id);
3775                    }
3776                    let merged =
3777                        with_child(&d, state, false, block.attrs.as_ref(), |expanded, state| {
3778                            let mut children = expanded.children;
3779                            if d.code.is_some() {
3780                                let table =
3781                                    if let [BlockNode::CodeBlock(code)] = children.as_mut_slice() {
3782                                        code.attrs.clone_from(&block.attrs);
3783                                        // `{table}` converts a csv/tsv code include as it does an authored fence (I1c).
3784                                        if selected_table_include(&d) {
3785                                            None
3786                                        } else {
3787                                            table_include(code, &[])
3788                                        }
3789                                    } else {
3790                                        None
3791                                    };
3792                                if let Some(table) = table {
3793                                    children[0] = BlockNode::Table(table);
3794                                }
3795                            }
3796                            merge_footnotes(
3797                                expanded.footnotes,
3798                                &mut children,
3799                                state,
3800                                expanded.file,
3801                                &expanded.reach,
3802                            );
3803                            Some(children)
3804                        });
3805                    if let (None, Some(id)) = (&merged, &held) {
3806                        let owner = state.owner;
3807                        state.reserve_id(id, owner);
3808                    }
3809                    merged
3810                });
3811            }
3812            ParsedDirective::InvalidLines(path) => {
3813                return state.at(position, |state| {
3814                    reject_invalid_lines(&path, state);
3815                    None
3816                });
3817            }
3818            ParsedDirective::InvalidTable(path, rule) => {
3819                return state.at(position, |state| {
3820                    reject_invalid_table(&path, rule, state);
3821                    None
3822                });
3823            }
3824            ParsedDirective::BadOption(part) => {
3825                return state.at(position, |state| {
3826                    state.warn(
3827                        "include-unknown-option",
3828                        format!("Unknown include option \"{part}\"."),
3829                    );
3830                    None
3831                });
3832            }
3833            ParsedDirective::DuplicateSection(part) => {
3834                return state.at(position, |state| {
3835                    state.warn(
3836                        "include-selection-conflict",
3837                        format!("Include directive cannot name two sections: \"{part}\"."),
3838                    );
3839                    None
3840                });
3841            }
3842            ParsedDirective::DuplicateOption(part) => {
3843                return state.at(position, |state| {
3844                    state.warn(
3845                        "include-duplicate-option",
3846                        format!("Include option \"{part}\" repeats an option given earlier."),
3847                    );
3848                    None
3849                });
3850            }
3851            ParsedDirective::RepeatedTableOption(path, part) => {
3852                return state.at(position, |state| {
3853                    let directive = Directive {
3854                        path,
3855                        section: None,
3856                        lines: None,
3857                        rows: None,
3858                        cols: None,
3859                        table_attrs: None,
3860                        region: None,
3861                        code: None,
3862                        shift: Shift::By(0),
3863                    };
3864                    let _ = resolve_child(&directive, state);
3865                    state.warn(
3866                        "include-duplicate-option",
3867                        format!("Include option \"{part}\" repeats an option given earlier."),
3868                    );
3869                    None
3870                });
3871            }
3872            ParsedDirective::NotADirective => {
3873                // A whole-paragraph directive that failed to parse was already
3874                // reported here; skip the inline scan so it is not warned twice.
3875                if is_directive_shaped(&source) {
3876                    // Recheck: a shaped-but-unparsable token may still carry a
3877                    // bad option worth reporting, which parse_full_directive
3878                    // only sees when the overall shape matched.
3879                    return None;
3880                }
3881            }
3882        }
3883    }
3884    expand_inlines(&mut block.children, state);
3885    None
3886}
3887
3888/// The parser ends a code include at its own line (PART 10 I2), so a `^ `
3889/// caption below a `{table}` include arrives as the next paragraph. Rejoin it
3890/// for `expand_paragraph`, which reads the caption after the directive line.
3891fn absorb_code_include_caption(blocks: &mut Vec<BlockNode>, i: usize) {
3892    let (Some(BlockNode::Paragraph(lead)), Some(BlockNode::Paragraph(next))) =
3893        (blocks.get(i), blocks.get(i + 1))
3894    else {
3895        return;
3896    };
3897    let caption = next.attrs.is_none()
3898        && matches!(next.children.first(), Some(InlineNode::Text(t)) if t.value.starts_with("^ "));
3899    let table = lead
3900        .attrs
3901        .as_ref()
3902        .is_some_and(|attrs| attrs.key_values.get("table").is_some_and(String::is_empty));
3903    // The caption slot allows no blank line or one before the `^ ` line.
3904    let near = match (lead.pos.as_ref(), next.pos.as_ref()) {
3905        (Some(lead), Some(next)) => next.start_line <= lead.end_line + 2,
3906        _ => true,
3907    };
3908    if !(caption && table && near)
3909        || !directive_source(&lead.children).is_some_and(|s| code_include_line(&s).is_some())
3910    {
3911        return;
3912    }
3913    let BlockNode::Paragraph(next) = blocks.remove(i + 1) else {
3914        return;
3915    };
3916    if let Some(BlockNode::Paragraph(lead)) = blocks.get_mut(i) {
3917        lead.children
3918            .push(InlineNode::SoftBreak(crate::ast::Break { pos: None }));
3919        lead.children.extend(next.children);
3920    }
3921}
3922
3923fn expand_blocks(blocks: &mut Vec<BlockNode>, state: &mut State<'_>) {
3924    // Spec I8: this block list is ONE container. Headings in it set the context
3925    // for later blocks and for containers nested inside it, but the entry value
3926    // is restored on exit so a CLOSED sibling container never sets context.
3927    let entry_context = state.context_level;
3928    let mut i = 0usize;
3929    while i < blocks.len() {
3930        absorb_code_include_caption(blocks, i);
3931        let mut replacement: Option<Vec<BlockNode>> = None;
3932        match &mut blocks[i] {
3933            BlockNode::Paragraph(p) => replacement = expand_paragraph(p, state),
3934            BlockNode::BlockQuote(b) => expand_blocks(&mut b.children, state),
3935            BlockNode::Directive(d) => expand_blocks(&mut d.children, state),
3936            BlockNode::Div(d) => expand_blocks(&mut d.children, state),
3937            BlockNode::Section(d) => expand_blocks(&mut d.children, state),
3938            BlockNode::Admonition(a) => expand_blocks(&mut a.children, state),
3939            BlockNode::FigureGroup(g) => expand_blocks(&mut g.children, state),
3940            BlockNode::List(l) => {
3941                for item in &mut l.items {
3942                    expand_blocks(&mut item.children, state);
3943                }
3944            }
3945            BlockNode::DefinitionList(dl) => {
3946                for item in &mut dl.items {
3947                    for def in &mut item.definitions {
3948                        expand_blocks(&mut def.children, state);
3949                    }
3950                }
3951            }
3952            BlockNode::Figure(fig) => {
3953                match &mut *fig.target {
3954                    FigureTarget::BlockQuote(b) => expand_blocks(&mut b.children, state),
3955                    FigureTarget::Paragraph(p) => expand_inlines(&mut p.children, state),
3956                    FigureTarget::Table(t) => {
3957                        if let Some(caption) = &mut t.caption {
3958                            expand_inlines(caption, state);
3959                        }
3960                        for row in &mut t.rows {
3961                            for cell in &mut row.cells {
3962                                expand_inlines(&mut cell.children, state);
3963                                if let Some(blocks) = &mut cell.blocks {
3964                                    expand_blocks(blocks, state);
3965                                }
3966                            }
3967                        }
3968                    }
3969                    _ => {}
3970                }
3971                expand_inlines(&mut fig.caption, state);
3972            }
3973            BlockNode::Heading(h) => {
3974                // An AUTO heading id is the slug of the heading's text, and the
3975                // parser derived it BEFORE this pass replaces that text. Note
3976                // whether the id still matches, so one an include invalidates
3977                // can be re-derived - `# {{ title.crv }}` kept `title-crv`
3978                // while its heading read "My Title" (I2). An EXPLICIT id does
3979                // not match its own text and is left alone.
3980                let auto_before = h
3981                    .attrs
3982                    .as_ref()
3983                    .and_then(|a| a.id.clone())
3984                    .filter(|id| *id == heading_slug(&h.children));
3985                expand_inlines(&mut h.children, state);
3986                if let Some(stale) = auto_before {
3987                    let derived = heading_slug(&h.children);
3988                    if derived != stale {
3989                        if let Some(attrs) = h.attrs.as_mut() {
3990                            attrs.id = Some(derived);
3991                        }
3992                    }
3993                }
3994                state.context_level = h.level as i32;
3995            }
3996            BlockNode::Table(t) => {
3997                if let Some(caption) = &mut t.caption {
3998                    expand_inlines(caption, state);
3999                }
4000                for row in &mut t.rows {
4001                    for cell in &mut row.cells {
4002                        expand_inlines(&mut cell.children, state);
4003                        if let Some(blocks) = &mut cell.blocks {
4004                            expand_blocks(blocks, state);
4005                        }
4006                    }
4007                }
4008            }
4009            // CodeBlock and RawBlock are VERBATIM (I9).
4010            _ => {}
4011        }
4012        if let Some(replacement) = replacement {
4013            let len = replacement.len();
4014            blocks.splice(i..i + 1, replacement);
4015            // The merged blocks are now part of THIS container, so a heading
4016            // they contribute at this level sets the context for what follows -
4017            // "the document as assembled" (I8).
4018            for merged in &blocks[i..i + len] {
4019                if let BlockNode::Heading(h) = merged {
4020                    state.context_level = h.level as i32;
4021                }
4022            }
4023            i += len;
4024        } else {
4025            i += 1;
4026        }
4027    }
4028    state.context_level = entry_context;
4029}
4030
4031// ---------------------------------------------------------------------------
4032// Entry point
4033// ---------------------------------------------------------------------------
4034
4035/// Expand processor-level `{{ … }}` include directives in an already-parsed AST.
4036///
4037/// With no resolver configured, directives remain ordinary text and no warnings
4038/// are emitted - the pinned core behavior.
4039pub fn expand_includes(doc: Document, source: &str, options: &IncludeOptions<'_>) -> IncludeResult {
4040    expand_includes_with_extensions(doc, source, options, &options.extensions)
4041}
4042
4043/// [`expand_includes`], parsing each child with `extensions` rather than the
4044/// set on `options`.
4045pub(crate) fn expand_includes_with_extensions(
4046    doc: Document,
4047    source: &str,
4048    options: &IncludeOptions<'_>,
4049    extensions: &[&dyn CarveExtension],
4050) -> IncludeResult {
4051    let mut doc = doc;
4052    let mut state = State {
4053        opts: options,
4054        extensions,
4055        warnings: Vec::new(),
4056        included: Vec::new(),
4057        reach: Vec::new(),
4058        max_depth: options.max_depth.unwrap_or(DEFAULT_MAX_DEPTH),
4059        max_bytes: options
4060            .max_bytes
4061            .unwrap_or_else(|| MIN_BUDGET.max(source.len().saturating_mul(8))),
4062        used_bytes: 0,
4063        max_resolver_calls: options
4064            .max_resolver_calls
4065            .unwrap_or(DEFAULT_MAX_RESOLVER_CALLS),
4066        resolver_calls: 0,
4067        max_warnings: options.max_warnings.unwrap_or(DEFAULT_MAX_WARNINGS),
4068        suppressed_warnings: 0,
4069        seen_rules: std::collections::BTreeSet::new(),
4070        warning_position: None,
4071        track_positions: doc_carries_positions(&doc),
4072        spent: None,
4073        stack: options.source_path.clone().into_iter().collect(),
4074        depth: 0,
4075        file: options.source_path.clone(),
4076        used_ids: HashMap::new(),
4077        inclusions: 0,
4078        pending_renames: Vec::new(),
4079        owner: 0,
4080        reservation_frames: Vec::new(),
4081        dependencies: Vec::new(),
4082        dep_index: HashMap::new(),
4083        footnotes: Vec::new(),
4084        context_level: 0,
4085    };
4086    // Recognition needs no extra parse, but a document whose source contains no
4087    // "{{" at all cannot hold a directive in any position, so the AST walk is
4088    // skipped outright. Directive-free documents stay at parse cost.
4089    if options.resolver.is_some() && source.contains("{{") {
4090        // Child footnotes are expanded while their child is active. Revisit
4091        // only root definitions here, so merged child directives are not
4092        // resolved again after their text and positions have been coalesced.
4093        let root_footnotes: HashSet<String> = doc.footnote_defs.keys().cloned().collect();
4094        // Parent explicit ids are claimed FIRST (I5: parent before child), so
4095        // an included duplicate is the one renamed - even against a parent
4096        // heading that appears after the include site.
4097        let mut claim = |attrs: &mut Option<Attrs>, _: Option<&[InlineNode]>| {
4098            if let Some(id) = attrs.as_ref().and_then(|a| a.id.as_deref()) {
4099                state.reserve_id(id, 0);
4100            }
4101        };
4102        for_each_id_site(&mut doc.children, &mut claim);
4103        for body in doc.footnote_defs.values_mut() {
4104            for_each_id_site(body, &mut claim);
4105        }
4106        state.footnotes.push(std::mem::take(&mut doc.footnote_defs));
4107        expand_blocks(&mut doc.children, &mut state);
4108        let mut defs = state
4109            .footnotes
4110            .pop()
4111            .expect("footnote stack push/pop are balanced");
4112        // Each footnote body is its own container, with no preceding heading.
4113        state.footnotes.push(BTreeMap::new());
4114        for (label, body) in &mut defs {
4115            if !root_footnotes.contains(label) {
4116                continue;
4117            }
4118            state.context_level = 0;
4119            expand_blocks(body, &mut state);
4120        }
4121        let nested = state
4122            .footnotes
4123            .pop()
4124            .expect("footnote stack push/pop are balanced");
4125        for (label, body) in nested {
4126            defs.entry(label).or_insert(body);
4127        }
4128        doc.footnote_defs = defs;
4129        finish_renames(&mut doc, &mut state);
4130    }
4131    IncludeResult {
4132        doc,
4133        warnings: state.warnings,
4134        suppressed_warnings: state.suppressed_warnings,
4135        dependencies: state.dependencies,
4136        charged_bytes: state.used_bytes,
4137        included: state.included,
4138    }
4139}
4140
4141/// Whether the include pass runs for a given render target (spec I15).
4142///
4143/// Only the Carve target opts out, and the reason is not performance: that
4144/// target writes the document back as Carve SOURCE, and expanding first returns
4145/// a different document, with every child inlined and the directives gone. The
4146/// writer already preserves a directive verbatim (I12); expanding before it runs
4147/// takes that away by another route.
4148///
4149/// Lives here rather than in the CLI so the rule has ONE home, read by the CLI
4150/// and by the include-conformance suite alike. carve-js kept it inline in its
4151/// CLI and inlined every child on `render --carve` as a result.
4152pub fn expands_for_target(target: crate::RenderTarget) -> bool {
4153    !matches!(target, crate::RenderTarget::Carve)
4154}
4155
4156// ---------------------------------------------------------------------------
4157// Filesystem resolver (spec I10)
4158// ---------------------------------------------------------------------------
4159
4160/// Default per-target read cap for [`FileSystemResolver`]: 4 MiB.
4161#[cfg(feature = "fs")]
4162pub const DEFAULT_MAX_FILE_BYTES: u64 = 4 * 1024 * 1024;
4163
4164/// Filesystem resolver with canonical root-containment checks, for TRUSTED
4165/// hosts (a CLI, a static-site build). Not for untrusted input.
4166///
4167/// Every candidate is canonicalized (symlinks resolved) and only then checked
4168/// against the canonical root. This is deliberately NOT a lexical ban on `..`,
4169/// which is wrong on both sides: TOO STRICT (a document in `chapters/`
4170/// including `../shared/glossary.crv` is a normal book layout whose canonical
4171/// target is inside the root) and TOO WEAK (a symlink inside the root pointing
4172/// out of it, or an absolute path, escapes with no `..` present at all).
4173#[cfg(feature = "fs")]
4174pub struct FileSystemResolver {
4175    root_real: PathBuf,
4176    allow_absolute: bool,
4177    max_file_bytes: Option<u64>,
4178}
4179
4180#[cfg(feature = "fs")]
4181impl FileSystemResolver {
4182    /// Canonicalizes `root` up front; fails if it does not exist, and refuses
4183    /// a spec that is not ABSOLUTE.
4184    ///
4185    /// A relative spec names no root: every canonicalizer resolves one against
4186    /// the process working directory, the value §19 forbids the root defaulting
4187    /// to. The test has to be on the CONFIGURED value - the canonicalized
4188    /// result is always absolute. A front end keeps the convenience by
4189    /// absolutizing its own flag before constructing this.
4190    pub fn new(root: impl AsRef<Path>) -> std::io::Result<Self> {
4191        let root = root.as_ref();
4192        if !root.is_absolute() {
4193            return Err(std::io::Error::new(
4194                std::io::ErrorKind::InvalidInput,
4195                format!(
4196                    "include containment root must be an absolute path, got {}",
4197                    root.display()
4198                ),
4199            ));
4200        }
4201        Ok(Self {
4202            root_real: std::fs::canonicalize(root)?,
4203            allow_absolute: false,
4204            max_file_bytes: Some(DEFAULT_MAX_FILE_BYTES),
4205        })
4206    }
4207
4208    /// Largest target this resolver will read. `None` removes the cap.
4209    pub fn with_max_file_bytes(mut self, bytes: Option<u64>) -> Self {
4210        self.max_file_bytes = bytes;
4211        self
4212    }
4213
4214    /// Allow absolute include paths. They are STILL subject to the same
4215    /// canonical containment check, so this only widens spelling, not reach.
4216    pub fn allow_absolute(mut self, allow: bool) -> Self {
4217        self.allow_absolute = allow;
4218        self
4219    }
4220
4221    /// Containment test over CANONICAL paths.
4222    ///
4223    /// `Path::strip_prefix` compares whole components, so a sibling directory
4224    /// legitimately named `..foo` (or `rootother` next to `root`) is not
4225    /// misread as an escape the way a string-prefix test would be. Both sides
4226    /// are already canonical, so no `..` component survives to be re-walked.
4227    fn contains(&self, candidate: &Path) -> bool {
4228        candidate.strip_prefix(&self.root_real).is_ok()
4229    }
4230
4231    /// Where `include_path` would land, before containment and before the
4232    /// target is looked for. `None` means this resolver will not look there at
4233    /// all, which today is an absolute path where those are not allowed.
4234    ///
4235    /// ONE ROOT PER EXPANSION (I10): relative paths resolve against the
4236    /// INCLUDING file, but containment is checked against the single top-level
4237    /// root. The root must NOT re-base per child, or a nested document could
4238    /// never reach a sibling directory of the project. The stack carries the
4239    /// canonical path of each ancestor, so a nested relative include resolves
4240    /// against its actual parent directory.
4241    fn candidate(&self, include_path: &str, ctx: &IncludeContext<'_>) -> Option<PathBuf> {
4242        let requested = Path::new(include_path);
4243        if requested.is_absolute() {
4244            if !self.allow_absolute {
4245                return None;
4246            }
4247            return Some(requested.to_path_buf());
4248        }
4249        let base = match ctx.stack.last() {
4250            Some(parent) => self
4251                .root_real
4252                .join(parent)
4253                .parent()
4254                .map(Path::to_path_buf)
4255                .unwrap_or_else(|| self.root_real.clone()),
4256            None => self.root_real.clone(),
4257        };
4258        Some(base.join(requested))
4259    }
4260}
4261
4262/// Whether `path` opens with a URI scheme, which names no place on this
4263/// filesystem for I11 to point at. The refusal itself is unchanged; only the
4264/// naming is withheld.
4265#[cfg(feature = "fs")]
4266fn is_uri(path: &str) -> bool {
4267    let Some(colon) = path.find(':') else {
4268        return false;
4269    };
4270    let scheme = &path[..colon];
4271    scheme.starts_with(|c: char| c.is_ascii_alphabetic())
4272        && scheme
4273            .chars()
4274            .all(|c| c.is_ascii_alphanumeric() || c == '+' || c == '-' || c == '.')
4275}
4276
4277#[cfg(feature = "fs")]
4278impl IncludeResolver for FileSystemResolver {
4279    fn resolve(
4280        &self,
4281        include_path: &str,
4282        ctx: &IncludeContext<'_>,
4283    ) -> Result<IncludeResolved, IncludeDenial> {
4284        let Some(candidate) = self.candidate(include_path, ctx) else {
4285            return Err(IncludeDenial::Denied);
4286        };
4287        // CONTAINMENT IS DECIDED FIRST, and lexically, so it does not depend on
4288        // the target existing: `mid/../../outside.crv` is outside the root
4289        // whether or not `mid` is there (markup-carve/carve#2021).
4290        if !self.contains(&lexical_real(&candidate)) {
4291            return Err(IncludeDenial::OutsideRoot);
4292        }
4293        // Checked again on the path actually read, in case a link changed
4294        // between the two canonicalizations.
4295        let real = std::fs::canonicalize(&candidate).map_err(|e| match e.kind() {
4296            std::io::ErrorKind::NotFound => IncludeDenial::NotFound,
4297            _ => IncludeDenial::Denied,
4298        })?;
4299        if !self.contains(&real) {
4300            return Err(IncludeDenial::OutsideRoot);
4301        }
4302        // SIZE IS CHECKED BEFORE THE READ, and the expansion budget cannot
4303        // stand in for it: the budget charges a target only once its source is
4304        // in hand, so without a cap here one oversized file is read into
4305        // memory in full before expansion refuses it.
4306        // ONLY A REGULAR FILE IS READ: a FIFO reports length 0, passes the
4307        // cap, and reading it then blocks until a writer opens it.
4308        let metadata = std::fs::metadata(&real).map_err(|_| IncludeDenial::Denied)?;
4309        if !metadata.is_file()
4310            || self
4311                .max_file_bytes
4312                .is_some_and(|limit| metadata.len() > limit)
4313        {
4314            return Err(IncludeDenial::Denied);
4315        }
4316        // Read through the CANONICAL path: it holds no symlink components, so
4317        // the check that just passed describes the bytes actually read. A
4318        // residual TOCTOU window remains if a directory component is swapped
4319        // between the two syscalls, which is why this resolver is for trusted
4320        // trees only.
4321        let bytes = std::fs::read(&real).map_err(|_| IncludeDenial::Denied)?;
4322        // PART 6 I7: ill-formed bytes in an INCLUDED file are non-text and
4323        // refused; only the document itself is replaced (CARVE-P0-023).
4324        let source = String::from_utf8(bytes).map_err(|_| IncludeDenial::NonText)?;
4325        Ok(IncludeResolved::with_id(
4326            source,
4327            real.to_string_lossy().into_owned(),
4328        ))
4329    }
4330
4331    /// I11: a target that is simply not there is named by where it would be,
4332    /// the path a host watches. One that would land outside the root is refused
4333    /// like any other escape and keeps the directive's spelling.
4334    fn unresolved_id(&self, include_path: &str, ctx: &IncludeContext<'_>) -> Option<String> {
4335        if is_uri(include_path) {
4336            return None;
4337        }
4338        let would_be = lexical_real(&self.candidate(include_path, ctx)?);
4339        if !self.contains(&would_be) || would_be.exists() {
4340            return None;
4341        }
4342        Some(would_be.to_string_lossy().into_owned())
4343    }
4344}
4345
4346/// `candidate` with its existing prefix canonicalized and the rest collapsed
4347/// lexically, so containment can be decided for a path that is not there.
4348#[cfg(feature = "fs")]
4349fn lexical_real(candidate: &Path) -> PathBuf {
4350    // The longest prefix that exists is canonicalized, which resolves the
4351    // symlinks actually on disk; what remains is pure arithmetic.
4352    let parts: Vec<std::ffi::OsString> = candidate
4353        .components()
4354        .map(|c| c.as_os_str().to_os_string())
4355        .collect();
4356    let mut split = parts.len();
4357    let canonical = loop {
4358        let prefix: PathBuf = parts[..split].iter().collect();
4359        if split == 0 {
4360            break PathBuf::new();
4361        }
4362        match std::fs::canonicalize(&prefix) {
4363            Ok(real) => break real,
4364            Err(_) => split -= 1,
4365        }
4366    };
4367    let mut out = canonical;
4368    for part in &parts[split..] {
4369        if part == ".." {
4370            out.pop();
4371        } else if part != "." {
4372            out.push(part);
4373        }
4374    }
4375    out
4376}