ndg_commonmark/processor/

core.rs

//! Core implementation of the Markdown processor.
//!
//! Main implementation of `MarkdownProcessor` and its methods focused on the
//! core rendering pipeline and configuration management.
#[expect(
  clippy::disallowed_types,
  reason = "Required for generic hasher abstraction"
)]
use std::collections::HashMap;
use std::{
  path::{Component, Path, PathBuf},
  sync::LazyLock,
};

use comrak::{
  Arena,
  nodes::{AstNode, NodeHeading, NodeValue},
  options::Options,
  parse_document,
};
use log::trace;
use markup5ever::local_name;
use regex::Regex;
use rustc_hash::FxHashMap;
use walkdir::WalkDir;

use super::{
  dom::safe_select,
  process::process_safe,
  types::{
    AstTransformer,
    MarkdownExtension,
    MarkdownOptions,
    MarkdownProcessor,
    PromptTransformer,
  },
};
use crate::{
  syntax::create_default_manager,
  types::{Header, MarkdownResult},
  utils,
};

static HEADER_ANCHOR_RE: LazyLock<Regex> = LazyLock::new(|| {
  Regex::new(r"<h([1-6])>(.*?)\s*\{#([a-zA-Z0-9_.-]+)\}(.*?)</h[1-6]>")
    .unwrap_or_else(|e| {
      log::error!("Failed to compile HEADER_ANCHOR_RE regex: {e}");
      utils::never_matching_regex().unwrap_or_else(|_| {
        #[expect(
          clippy::expect_used,
          reason = "This pattern is guaranteed to be valid"
        )]
        Regex::new(r"[^\s\S]")
          .expect("regex pattern [^\\s\\S] should always compile")
      })
    })
});

static HEADER_NO_ID_RE: LazyLock<Regex> = LazyLock::new(|| {
  Regex::new(r"<h([1-6])>(.*?)</h[1-6]>").unwrap_or_else(|e| {
    log::error!("Failed to compile HEADER_NO_ID_RE regex: {e}");
    utils::never_matching_regex().unwrap_or_else(|_| {
      #[expect(
        clippy::expect_used,
        reason = "This pattern is guaranteed to be valid"
      )]
      Regex::new(r"[^\s\S]")
        .expect("regex pattern [^\\s\\S] should always compile")
    })
  })
});

static HTML_TAG_RE: LazyLock<Regex> = LazyLock::new(|| {
  Regex::new(r"<[^>]+>").unwrap_or_else(|e| {
    log::error!("Failed to compile HTML_TAG_RE regex: {e}");
    utils::never_matching_regex().unwrap_or_else(|_| {
      #[expect(
        clippy::expect_used,
        reason = "This pattern is guaranteed to be valid"
      )]
      Regex::new(r"[^\s\S]")
        .expect("regex pattern [^\\s\\S] should always compile")
    })
  })
});

impl MarkdownProcessor {
  /// Create a new `MarkdownProcessor` with the given options.
  #[must_use]
  pub fn new(options: MarkdownOptions) -> Self {
    let manpage_urls = options.manpage_urls.clone();

    let syntax_manager = if options.highlight_code {
      match create_default_manager(
        options
          .syntax_queries_path
          .as_deref()
          .map(std::path::Path::new),
      ) {
        Ok(manager) => {
          log::info!("Syntax highlighting initialized successfully");
          Some(manager)
        },
        Err(e) => {
          log::error!("Failed to initialize syntax highlighting: {e}");
          log::warn!(
            "Continuing without syntax highlighting - code blocks will not be \
             highlighted"
          );
          None
        },
      }
    } else {
      None
    };

    Self {
      options,
      manpage_urls,
      syntax_manager,
      base_dir: std::path::PathBuf::from("."),
    }
  }

  /// Access processor options.
  #[must_use]
  pub const fn options(&self) -> &MarkdownOptions {
    &self.options
  }

  /// Set the base directory for resolving relative file paths.
  #[must_use]
  pub fn with_base_dir(mut self, base_dir: &std::path::Path) -> Self {
    self.base_dir = base_dir.to_path_buf();
    self
  }

  /// Check if a specific feature is enabled.
  #[must_use]
  pub const fn has_feature(&self, feature: ProcessorFeature) -> bool {
    match feature {
      ProcessorFeature::Gfm => self.options.gfm,
      ProcessorFeature::Nixpkgs => self.options.nixpkgs,
      ProcessorFeature::SyntaxHighlighting => self.options.highlight_code,
      ProcessorFeature::ManpageUrls => self.manpage_urls.is_some(),
    }
  }

  /// Get the manpage URLs mapping for use with standalone functions.
  #[must_use]
  pub const fn manpage_urls(&self) -> Option<&FxHashMap<String, String>> {
    self.manpage_urls.as_ref()
  }

  /// Highlight all code blocks in HTML using the configured syntax highlighter
  #[must_use]
  pub fn highlight_codeblocks(&self, html: &str) -> String {
    use kuchikikiki::parse_html;
    use tendril::TendrilSink;

    if !self.options.highlight_code || self.syntax_manager.is_none() {
      return html.to_string();
    }

    let document = parse_html().one(html);
    self.highlight_codeblocks_document(&document);

    let mut buf = Vec::new();
    if let Err(e) = document.serialize(&mut buf) {
      log::warn!("DOM serialization failed: {e:?}");
      return html.to_string(); // Return original HTML if serialization fails
    }
    String::from_utf8(buf).unwrap_or_else(|_| html.to_string())
  }

  /// Handle hard tabs in code blocks according to configuration
  fn handle_hardtabs(&self, code: &str) -> String {
    use super::types::TabStyle;

    // Check if there are any hard tabs
    if !code.contains('\t') {
      return code.to_string();
    }

    match self.options.tab_style {
      // Do nothing
      TabStyle::None => code.to_string(),

      // Warn, but do nothing.
      TabStyle::Warn => {
        log::warn!(
          "Hard tabs detected in code block. Consider using spaces for \
           consistency. Tools like editorconfig may help you normalize spaces \
           in your documents."
        );
        code.to_string()
      },

      // Do not warn, only inform in debug mode. Then return
      // the updated code.
      TabStyle::Normalize => {
        log::debug!("Replacing hard tabs with spaces");
        code.replace('\t', "  ")
      },
    }
  }

  /// Process hard tabs in code blocks within markdown content
  fn process_hardtabs(&self, markdown: &str) -> String {
    use super::types::TabStyle;
    use crate::utils::codeblock::FenceTracker;

    // If no tab handling is needed, return as-is
    if self.options.tab_style == TabStyle::None {
      return markdown.to_string();
    }

    let mut result = String::with_capacity(markdown.len());
    let mut lines = markdown.lines().peekable();
    let mut tracker = FenceTracker::new();

    while let Some(line) = lines.next() {
      tracker = tracker.process_line(line);

      // Only replace tabs inside fenced code blocks
      let processed_line = if tracker.in_code_block() && line.contains('\t') {
        self.handle_hardtabs(line)
      } else {
        line.to_string()
      };

      result.push_str(&processed_line);

      // Add newline unless this is the last line
      if lines.peek().is_some() {
        result.push('\n');
      }
    }

    result
  }

  /// Highlight code using the configured syntax highlighter, returns HTML
  /// string
  fn highlight_code_html(&self, code: &str, language: &str) -> Option<String> {
    if !self.options.highlight_code {
      return None;
    }

    let syntax_manager = self.syntax_manager.as_ref()?;

    syntax_manager
      .highlight_code(code, language, self.options.highlight_theme.as_deref())
      .ok()
  }

  /// Render Markdown to HTML, extracting headers and title.
  #[must_use]
  pub fn render(&self, markdown: &str) -> MarkdownResult {
    let (preprocessed, included_files) = self.preprocess(markdown);
    let arena = Arena::new();
    let options = self.comrak_options();
    let root = parse_document(&arena, &preprocessed, &options);
    let (headers, title) = Self::extract_headers_from_ast(root, &preprocessed);
    let html = self.process_html_pipeline_from_ast(root, &options);

    MarkdownResult {
      html,
      headers,
      title,
      included_files,
    }
  }

  /// Render Markdown to HTML with anchor validation.
  ///
  /// Same as [`render()`](Self::render), but additionally validates that all
  /// heading anchor IDs are unique. Returns an error if duplicates are found.
  ///
  /// # Errors
  ///
  /// Returns `Err(DuplicateAnchorError)` if any two headings share the same
  /// anchor ID.
  pub fn render_checked(
    &self,
    markdown: &str,
  ) -> Result<MarkdownResult, crate::types::DuplicateAnchorError> {
    let result = self.render(markdown);
    crate::types::validate_rendered_anchor_ids(&result.headers, &result.html)?;
    Ok(result)
  }

  fn process_html_pipeline_from_ast<'a>(
    &self,
    root: &'a AstNode<'a>,
    options: &Options<'a>,
  ) -> String {
    let mut html = Self::format_html(root, options);
    self.process_rendered_html(&mut html);
    html
  }

  fn process_rendered_html(&self, html: &mut String) {
    // Apply feature-specific post-processing
    if cfg!(feature = "ndg-flavored") {
      #[cfg(feature = "ndg-flavored")]
      {
        *html = super::extensions::process_option_references(
          html,
          self.options.valid_options.as_ref(),
        );
      }
    }

    if self.options.nixpkgs {
      *html = self.process_manpage_references_html(html);
    }

    *html = self.kuchiki_postprocess(html);
  }

  /// Preprocess the markdown content with all enabled transformations.
  fn preprocess(
    &self,
    content: &str,
  ) -> (String, Vec<crate::types::IncludedFile>) {
    let mut processed = content.to_string();
    let mut included_files = Vec::new();

    // Process MyST-style autolinks first
    processed = super::extensions::process_myst_autolinks(&processed);

    // Handle hard tabs in code blocks
    processed = self.process_hardtabs(&processed);

    if self.options.nixpkgs {
      let (content, files) = self.apply_nixpkgs_preprocessing(&processed);
      processed = content;
      included_files = files;
    }

    #[cfg(any(feature = "nixpkgs", feature = "ndg-flavored"))]
    if self.options.nixpkgs || cfg!(feature = "ndg-flavored") {
      processed = super::extensions::process_role_markup(
        &processed,
        self.manpage_urls.as_ref(),
        self.options.auto_link_options,
        self.options.valid_options.as_ref(),
      );
    }

    #[cfg(feature = "wiki")]
    {
      processed = super::extensions::process_wikilinks(&processed);
    }

    (processed, included_files)
  }

  /// Apply Nixpkgs-specific preprocessing steps.
  #[cfg(feature = "nixpkgs")]
  fn apply_nixpkgs_preprocessing(
    &self,
    content: &str,
  ) -> (String, Vec<crate::types::IncludedFile>) {
    let (with_includes, included_files) =
      match super::extensions::process_file_includes(content, &self.base_dir, 0)
      {
        Ok(result) => result,
        Err(e) => {
          log::warn!(
            "File include processing failed: {e}. Continuing without includes."
          );
          (content.to_string(), Vec::new())
        },
      };
    let with_blocks = super::extensions::process_block_elements(&with_includes);
    let with_spans = super::extensions::process_bracketed_spans(&with_blocks);
    let processed = super::extensions::process_inline_anchors(&with_spans);
    (processed, included_files)
  }

  /// Apply Nixpkgs-specific preprocessing steps (no-op when feature disabled).
  #[cfg(not(feature = "nixpkgs"))]
  fn apply_nixpkgs_preprocessing(
    &self,
    content: &str,
  ) -> (String, Vec<crate::types::IncludedFile>) {
    (content.to_string(), Vec::new())
  }

  /// Extract headers and title from the markdown content.
  #[must_use]
  pub fn extract_headers(
    &self,
    content: &str,
  ) -> (Vec<Header>, Option<String>) {
    let arena = Arena::new();
    let options = self.comrak_options();
    let root = parse_document(&arena, content, &options);
    Self::extract_headers_from_ast(root, content)
  }

  /// Extract headings from the already-rendered Comrak tree.
  fn extract_headers_from_ast<'a>(
    root: &'a AstNode<'a>,
    source: &str,
  ) -> (Vec<Header>, Option<String>) {
    let mut headers = Vec::new();
    let mut title = None;
    let ignored_lines = admonition_lines(source);
    for node in root.descendants() {
      let NodeValue::Heading(NodeHeading { level, .. }) =
        &node.data.borrow().value
      else {
        continue;
      };
      let line = node.data.borrow().sourcepos.start.line;
      if ignored_lines.get(line).copied().unwrap_or(false) {
        continue;
      }
      let mut text = String::new();
      let mut explicit_id = None;
      for child in node.children() {
        match &child.data.borrow().value {
          NodeValue::Text(value) => text.push_str(value),
          NodeValue::Code(value) => text.push_str(&value.literal),
          NodeValue::Link(..)
          | NodeValue::Emph
          | NodeValue::Strong
          | NodeValue::Subscript
          | NodeValue::Strikethrough
          | NodeValue::Superscript
          | NodeValue::FootnoteReference(..) => {
            text.push_str(&extract_inline_text(child));
          },
          NodeValue::HtmlInline(value) => {
            if let Some(start) = value.find("{#")
              && let Some(end) = value[start..].find('}')
            {
              explicit_id = Some(value[start + 2..start + end].to_string());
            }
          },
          _ => {},
        }
      }
      let trimmed = text.trim_end();
      let (text, id) = if let Some(start) = trimmed.rfind("{#")
        && let Some(end) = trimmed[start..].find('}')
      {
        (
          trimmed[..start].trim_end().to_string(),
          trimmed[start + 2..start + end].to_string(),
        )
      } else {
        (
          text.clone(),
          explicit_id.unwrap_or_else(|| slugify_heading(&text)),
        )
      };
      if *level == 1 && title.is_none() {
        title = Some(text.clone());
      }
      headers.push((line, Header {
        text,
        level: *level,
        id,
      }));
    }
    let lines: Vec<_> = source.lines().collect();
    let mut fence = utils::codeblock::FenceTracker::new();
    for (index, line) in lines.iter().enumerate() {
      fence = fence.process_line(line);
      if ignored_lines.get(index + 1).copied().unwrap_or(false) {
        continue;
      }
      if fence.in_code_block() {
        continue;
      }
      let trimmed = line.trim();
      if trimmed.starts_with('#')
        || lines
          .get(index + 1)
          .is_some_and(|next| is_setext_heading_underline(next.trim()))
      {
        continue;
      }
      if let Some(start) = trimmed.rfind("{#")
        && let Some(end) = trimmed[start..].find('}')
      {
        let text = trimmed[..start].trim_end();
        let id = &trimmed[start + 2..start + end];
        headers.push((index + 1, Header {
          text:  text.to_string(),
          level: 2,
          id:    id.to_string(),
        }));
      }
    }
    headers.sort_by_key(|(line, _)| *line);
    (
      headers.into_iter().map(|(_, header)| header).collect(),
      title,
    )
  }

  fn format_html<'a>(root: &'a AstNode<'a>, options: &Options<'a>) -> String {
    // Apply AST transformations
    let prompt_transformer = PromptTransformer;
    prompt_transformer.transform(root);

    let mut html_output = String::new();
    if let Err(e) = comrak::format_html(root, options, &mut html_output) {
      log::error!("Failed to format HTML: {e}");
    }

    // Post-process HTML to handle header anchors
    Self::process_header_anchors_html(&html_output)
  }

  /// Process header anchors in HTML by finding `{#id}` syntax and converting to
  /// proper id attributes. Also adds auto-generated IDs to headers without
  /// explicit anchors.
  fn process_header_anchors_html(html: &str) -> String {
    // First pass: explicit {#id} syntax
    let result = HEADER_ANCHOR_RE
      .replace_all(html, |caps: &regex::Captures| {
        let level = &caps[1];
        let prefix = &caps[2];
        let id = &caps[3];
        let suffix = &caps[4];
        format!("<h{level} id=\"{id}\">{prefix}{suffix}</h{level}>")
      })
      .to_string();

    // Second pass: add auto-generated IDs to headers without id attribute
    HEADER_NO_ID_RE
      .replace_all(&result, |caps: &regex::Captures| {
        let level = &caps[1];
        let content = &caps[2];
        // Strip HTML tags and slugify the text content
        let text_only = HTML_TAG_RE.replace_all(content, "");
        let id = utils::slugify(&text_only);
        if id.is_empty() {
          // If slugify produces empty string, keep header without id
          format!("<h{level}>{content}</h{level}>")
        } else {
          format!("<h{level} id=\"{id}\">{content}</h{level}>")
        }
      })
      .to_string()
  }

  /// Build comrak options from `MarkdownOptions` and feature flags.
  fn comrak_options(&self) -> Options<'_> {
    let mut options = Options::default();
    // Markdown features present in GFM.
    if self.options.gfm {
      options.extension.table = true;
      options.extension.footnotes = true;
      options.extension.strikethrough = true;
      options.extension.tasklist = true;
      options.extension.superscript = true;
      options.extension.autolink = true;
    }

    for extension in &self.options.extensions {
      match extension {
        MarkdownExtension::Alerts => options.extension.alerts = true,
        MarkdownExtension::Autolink => options.extension.autolink = true,
        MarkdownExtension::BlockDirective => {
          options.extension.block_directive = true;
        },
        MarkdownExtension::CjkFriendlyEmphasis => {
          options.extension.cjk_friendly_emphasis = true;
        },
        MarkdownExtension::DescriptionLists => {
          options.extension.description_lists = true;
        },
        MarkdownExtension::Footnotes => options.extension.footnotes = true,
        MarkdownExtension::Greentext => options.extension.greentext = true,
        MarkdownExtension::Highlight => options.extension.highlight = true,
        MarkdownExtension::Insert => options.extension.insert = true,
        MarkdownExtension::InlineFootnotes => {
          options.extension.inline_footnotes = true;
        },
        MarkdownExtension::MathCode => options.extension.math_code = true,
        MarkdownExtension::MathDollars => {
          options.extension.math_dollars = true;
        },
        MarkdownExtension::MathLatex => options.extension.math_latex = true,
        MarkdownExtension::MultilineBlockQuotes => {
          options.extension.multiline_block_quotes = true;
        },
        MarkdownExtension::Spoiler => options.extension.spoiler = true,
        MarkdownExtension::Strikethrough => {
          options.extension.strikethrough = true;
        },
        MarkdownExtension::Subscript => options.extension.subscript = true,
        MarkdownExtension::Subtext => options.extension.subtext = true,
        MarkdownExtension::Superscript => options.extension.superscript = true,
        MarkdownExtension::Table => options.extension.table = true,
        MarkdownExtension::Tasklist => options.extension.tasklist = true,
        MarkdownExtension::Underline => options.extension.underline = true,
        MarkdownExtension::WikilinksTitleAfterPipe => {
          options.extension.wikilinks_title_after_pipe = true;
        },
        MarkdownExtension::WikilinksTitleBeforePipe => {
          options.extension.wikilinks_title_before_pipe = true;
        },
      }
    }

    // Enable unsafe HTML references. This is not a security concern
    // as all input is assumed to be trusted.
    options.render.r#unsafe = true;

    // Enable description lists but keep custom header processing
    options.extension.header_id_prefix = None;
    options.extension.description_lists = true;
    options
  }

  /// Post-process HTML to enhance manpage references with URL links.
  #[cfg(feature = "nixpkgs")]
  fn process_manpage_references_html(&self, html: &str) -> String {
    super::extensions::process_manpage_references(
      html,
      self.manpage_urls.as_ref(),
    )
  }

  /// Post-process HTML to enhance manpage references (no-op when feature
  /// disabled).
  #[cfg(not(feature = "nixpkgs"))]
  fn process_manpage_references_html(&self, html: &str) -> String {
    html.to_string()
  }

  /// HTML post-processing using kuchiki DOM manipulation.
  fn kuchiki_postprocess(&self, html: &str) -> String {
    // Use a standalone function to avoid borrowing issues
    kuchiki_postprocess_html(html, |document| {
      self.highlight_codeblocks_document(document);
      Self::apply_dom_transformations(document);
    })
  }

  fn highlight_codeblocks_document(&self, document: &kuchikikiki::NodeRef) {
    if !self.options.highlight_code || self.syntax_manager.is_none() {
      return;
    }
    let mut code_blocks = Vec::new();
    for code_node in safe_select(document, "pre > code") {
      if let Some(element) = code_node.as_element() {
        let language = element
          .attributes
          .borrow()
          .get("class")
          .and_then(|class| class.strip_prefix("language-"))
          .unwrap_or("text")
          .to_string();
        if let Some(pre_node) = code_node.parent() {
          code_blocks.push((pre_node, code_node.text_contents(), language));
        }
      }
    }
    for (pre_node, code, language) in code_blocks {
      if let Some(highlighted) = self.highlight_code_html(&code, &language) {
        use tendril::TendrilSink;
        let replacement = kuchikikiki::parse_html().one(format!(
          r#"<pre class="highlight"><code class="language-{language}">{highlighted}</code></pre>"#
        ));
        pre_node.insert_after(replacement);
        pre_node.detach();
      }
    }
  }

  /// Apply all DOM transformations to the parsed HTML document.
  fn apply_dom_transformations(document: &kuchikikiki::NodeRef) {
    Self::process_list_item_id_markers(document);
    Self::process_header_anchor_comments(document);
    Self::process_list_item_inline_anchors(document);
    Self::process_paragraph_inline_anchors(document);
    Self::process_remaining_inline_anchors(document);
    Self::process_markdown_links(document);
    Self::process_option_anchor_links(document);
    Self::process_empty_auto_links(document);
  }

  /// Process list item ID markers: <li><!-- nixos-anchor-id:ID -->
  fn process_list_item_id_markers(document: &kuchikikiki::NodeRef) {
    let mut to_modify = Vec::new();

    for comment in document.inclusive_descendants() {
      if let Some(comment_node) = comment.as_comment() {
        let comment_text = comment_node.borrow();
        if let Some(id_start) = comment_text.find("nixos-anchor-id:") {
          let id = comment_text[id_start + 16..].trim();
          if !id.is_empty()
            && id
              .chars()
              .all(|c| c.is_alphanumeric() || c == '-' || c == '_')
          {
            // Check if this comment is inside an <li> element
            if let Some(parent) = comment.parent()
              && let Some(element) = parent.as_element()
              && element.name.local.as_ref() == "li"
            {
              to_modify.push((comment.clone(), id.to_string()));
            }
          }
        }
      }
    }

    for (comment_node, id) in to_modify {
      let span = kuchikikiki::NodeRef::new_element(
        markup5ever::QualName::new(
          None,
          markup5ever::ns!(html),
          local_name!("span"),
        ),
        vec![
          (
            kuchikikiki::ExpandedName::new("", "id"),
            kuchikikiki::Attribute {
              prefix: None,
              value:  id,
            },
          ),
          (
            kuchikikiki::ExpandedName::new("", "class"),
            kuchikikiki::Attribute {
              prefix: None,
              value:  "nixos-anchor".into(),
            },
          ),
        ],
      );
      comment_node.insert_after(span);
      comment_node.detach();
    }
  }

  /// Process header anchors with comments: <h1>text<!-- anchor: id --></h1>
  fn process_header_anchor_comments(document: &kuchikikiki::NodeRef) {
    let mut to_modify = Vec::new();

    for comment in document.inclusive_descendants() {
      if let Some(comment_node) = comment.as_comment() {
        let comment_text = comment_node.borrow();
        if let Some(anchor_start) = comment_text.find("anchor:") {
          let id = comment_text[anchor_start + 7..].trim();
          if !id.is_empty()
            && id
              .chars()
              .all(|c| c.is_alphanumeric() || c == '-' || c == '_')
          {
            // Check if this comment is inside a header element
            if let Some(parent) = comment.parent()
              && let Some(element) = parent.as_element()
              && matches!(
                element.name.local.as_ref(),
                "h1" | "h2" | "h3" | "h4" | "h5" | "h6"
              )
            {
              to_modify.push((parent.clone(), comment.clone(), id.to_string()));
            }
          }
        }
      }
    }

    for (header_element, comment_node, id) in to_modify {
      if let Some(element) = header_element.as_element() {
        element
          .attributes
          .borrow_mut()
          .insert(local_name!("id"), id);
        comment_node.detach();
      }
    }
  }

  /// Process remaining inline anchors in list items: <li>[]{#id}content</li>
  fn process_list_item_inline_anchors(document: &kuchikikiki::NodeRef) {
    for li_node in safe_select(document, "li") {
      let li_element = li_node;

      // Check if this list item contains code elements
      let has_code = !safe_select(&li_element, "code, pre").is_empty();
      if has_code {
        continue; // Skip list items with code blocks
      }

      let text_content = li_element.text_contents();

      if let Some(anchor_start) = text_content.find("[]{#")
        && let Some(anchor_end) = text_content[anchor_start..].find('}')
      {
        let id = &text_content[anchor_start + 4..anchor_start + anchor_end];
        if !id.is_empty()
          && id
            .chars()
            .all(|c| c.is_alphanumeric() || c == '-' || c == '_' || c == '.')
        {
          let remaining_content =
            &text_content[anchor_start + anchor_end + 1..];

          // Clear current content and rebuild
          for child in li_element.children() {
            child.detach();
          }

          let span = kuchikikiki::NodeRef::new_element(
            markup5ever::QualName::new(
              None,
              markup5ever::ns!(html),
              local_name!("span"),
            ),
            vec![
              (
                kuchikikiki::ExpandedName::new("", "id"),
                kuchikikiki::Attribute {
                  prefix: None,
                  value:  id.into(),
                },
              ),
              (
                kuchikikiki::ExpandedName::new("", "class"),
                kuchikikiki::Attribute {
                  prefix: None,
                  value:  "nixos-anchor".into(),
                },
              ),
            ],
          );
          li_element.append(span);
          if !remaining_content.is_empty() {
            li_element
              .append(kuchikikiki::NodeRef::new_text(remaining_content));
          }
        }
      }
    }
  }

  /// Process inline anchors in paragraphs: <p>[]{#id}content</p>
  fn process_paragraph_inline_anchors(document: &kuchikikiki::NodeRef) {
    for p_node in safe_select(document, "p") {
      let p_element = p_node;

      // Check if this paragraph contains code elements
      let has_code = !safe_select(&p_element, "code, pre").is_empty();
      if has_code {
        continue; // Skip paragraphs with code blocks
      }

      let text_content = p_element.text_contents();

      if let Some(anchor_start) = text_content.find("[]{#")
        && let Some(anchor_end) = text_content[anchor_start..].find('}')
      {
        let id = &text_content[anchor_start + 4..anchor_start + anchor_end];
        if !id.is_empty()
          && id
            .chars()
            .all(|c| c.is_alphanumeric() || c == '-' || c == '_' || c == '.')
        {
          let remaining_content =
            &text_content[anchor_start + anchor_end + 1..];

          // Clear current content and rebuild
          for child in p_element.children() {
            child.detach();
          }

          let span = kuchikikiki::NodeRef::new_element(
            markup5ever::QualName::new(
              None,
              markup5ever::ns!(html),
              local_name!("span"),
            ),
            vec![
              (
                kuchikikiki::ExpandedName::new("", "id"),
                kuchikikiki::Attribute {
                  prefix: None,
                  value:  id.into(),
                },
              ),
              (
                kuchikikiki::ExpandedName::new("", "class"),
                kuchikikiki::Attribute {
                  prefix: None,
                  value:  "nixos-anchor".into(),
                },
              ),
            ],
          );
          p_element.append(span);
          if !remaining_content.is_empty() {
            p_element.append(kuchikikiki::NodeRef::new_text(remaining_content));
          }
        }
      }
    }
  }

  /// Process remaining standalone inline anchors throughout the document
  fn process_remaining_inline_anchors(document: &kuchikikiki::NodeRef) {
    let mut text_nodes_to_process = Vec::new();

    for node in document.inclusive_descendants() {
      if let Some(text_node) = node.as_text() {
        // Check if this text node is inside a code block
        let mut parent = node.parent();
        let mut in_code = false;
        while let Some(p) = parent {
          if let Some(element) = p.as_element()
            && (element.name.local == local_name!("code")
              || element.name.local == local_name!("pre"))
          {
            in_code = true;
            break;
          }
          parent = p.parent();
        }

        // Only process if not in code
        if !in_code {
          let text_content = text_node.borrow().clone();
          if text_content.contains("[]{#") {
            text_nodes_to_process.push((node.clone(), text_content));
          }
        }
      }
    }

    for (text_node, text_content) in text_nodes_to_process {
      let mut last_end = 0;
      let mut new_children = Vec::new();

      // Simple pattern matching for []{#id}
      let chars = text_content.chars().collect::<Vec<_>>();
      let mut i = 0;
      while i < chars.len() {
        if i + 4 < chars.len()
          && chars[i] == '['
          && chars[i + 1] == ']'
          && chars[i + 2] == '{'
          && chars[i + 3] == '#'
        {
          // Found start of anchor pattern
          let anchor_start = i;
          i += 4; // skip "[]{#"

          let mut id = String::new();
          while i < chars.len() && chars[i] != '}' {
            if chars[i].is_alphanumeric()
              || chars[i] == '-'
              || chars[i] == '_'
              || chars[i] == '.'
            {
              id.push(chars[i]);
              i += 1;
            } else {
              break;
            }
          }

          if i < chars.len() && chars[i] == '}' && !id.is_empty() {
            // Valid anchor found
            let anchor_end = i + 1;

            // Add text before anchor
            if anchor_start > last_end {
              new_children.push(kuchikikiki::NodeRef::new_text(
                chars[last_end..anchor_start].iter().collect::<String>(),
              ));
            }

            // Add span element
            let span = kuchikikiki::NodeRef::new_element(
              markup5ever::QualName::new(
                None,
                markup5ever::ns!(html),
                local_name!("span"),
              ),
              vec![
                (
                  kuchikikiki::ExpandedName::new("", "id"),
                  kuchikikiki::Attribute {
                    prefix: None,
                    value:  id,
                  },
                ),
                (
                  kuchikikiki::ExpandedName::new("", "class"),
                  kuchikikiki::Attribute {
                    prefix: None,
                    value:  "nixos-anchor".into(),
                  },
                ),
              ],
            );
            new_children.push(span);

            last_end = anchor_end;
            i = anchor_end;
          } else {
            i += 1;
          }
        } else {
          i += 1;
        }
      }

      // Add remaining text
      if last_end < chars.len() {
        let after_text: String = chars[last_end..].iter().collect();
        if !after_text.is_empty() {
          new_children.push(kuchikikiki::NodeRef::new_text(after_text));
        }
      }

      // Replace text node if we found anchors
      if !new_children.is_empty() {
        for child in new_children {
          text_node.insert_before(child);
        }
        text_node.detach();
      }
    }
  }

  /// Process empty auto-links: [](#anchor) -> <a href="#anchor">Anchor</a>
  fn process_empty_auto_links(document: &kuchikikiki::NodeRef) {
    for link_node in safe_select(document, "a") {
      let link_element = link_node;
      if let Some(element) = link_element.as_element() {
        let href = element
          .attributes
          .borrow()
          .get(local_name!("href"))
          .map(std::string::ToString::to_string);
        let text_content = link_element.text_contents();
        // An element such as an image is content, although it has no text
        let has_element = link_element
          .children()
          .any(|child| child.as_element().is_some());

        if let Some(href_value) = href
          && href_value.starts_with('#')
          && ((text_content.trim().is_empty() && !has_element)
            || text_content.trim() == "{{ANCHOR}}")
        {
          // Clear placeholder text if present
          if text_content.trim() == "{{ANCHOR}}" {
            for child in link_element.children() {
              child.detach();
            }
          }
          // Empty link with anchor - add humanized text
          let display_text = humanize_anchor(&href_value);
          link_element.append(kuchikikiki::NodeRef::new_text(display_text));
        }
      }
    }
  }

  /// Process option anchor links: [](#opt-option.path) -> link to options.html
  fn process_option_anchor_links(document: &kuchikikiki::NodeRef) {
    let mut to_modify = Vec::new();

    // Collect all option anchor links first
    for link_node in safe_select(document, "a[href^='#opt-']") {
      let link_element = link_node;
      if let Some(element) = link_element.as_element() {
        let href = element
          .attributes
          .borrow()
          .get(local_name!("href"))
          .map(std::string::ToString::to_string);
        let text_content = link_element.text_contents();

        if let Some(href_value) = href
          && href_value.starts_with("#opt-")
        {
          let option_anchor = href_value[1..].to_string(); // remove the leading #
          let needs_text_replacement = text_content.trim().is_empty()
            || text_content.trim() == "{{ANCHOR}}";
          to_modify.push((
            link_element.clone(),
            option_anchor,
            needs_text_replacement,
          ));
        }
      }
    }

    // Apply modifications
    for (link_element, option_anchor, needs_text_replacement) in to_modify {
      if let Some(element) = link_element.as_element() {
        let new_href = format!("options.html#{option_anchor}");
        element
          .attributes
          .borrow_mut()
          .insert(local_name!("href"), new_href);

        if needs_text_replacement {
          // Clear existing content
          for child in link_element.children() {
            child.detach();
          }

          // Extract option name from anchor
          // opt-services-nginx-enable -> services.nginx.enable
          if let Some(option_path) = option_anchor.strip_prefix("opt-") {
            let option_name = option_path.replace('-', ".");
            link_element.append(kuchikikiki::NodeRef::new_text(option_name));
          }
        }
      }
    }
  }

  /// Process markdown file links: convert .md hrefs to .html
  fn process_markdown_links(document: &kuchikikiki::NodeRef) {
    for link_node in safe_select(document, "a") {
      let link_element = link_node;
      if let Some(element) = link_element.as_element() {
        let href = element
          .attributes
          .borrow()
          .get(local_name!("href"))
          .map(std::string::ToString::to_string);

        if let Some(href_value) = href {
          // Only process relative links ending in .md (not absolute URLs, not
          // anchors)
          if !href_value.starts_with("http://")
            && !href_value.starts_with("https://")
            && !href_value.starts_with('#')
            && !href_value.starts_with("mailto:")
          {
            // Split off fragment (#) and query (?) to check the path extension
            let (path_part, suffix) = href_value
              .find(['#', '?'])
              .map_or((href_value.as_str(), ""), |idx| {
                href_value.split_at(idx)
              });

            if std::path::Path::new(path_part)
              .extension()
              .is_some_and(|ext| ext.eq_ignore_ascii_case("md"))
            {
              let new_href =
                format!("{}.html{}", &path_part[..path_part.len() - 3], suffix);
              element
                .attributes
                .borrow_mut()
                .insert(local_name!("href"), new_href);
            }
          }
        }
      }
    }
  }
}

/// Convert an anchor ID to human-readable text.
fn humanize_anchor(anchor: &str) -> String {
  let cleaned = anchor.trim_start_matches('#');
  let without_prefix = cleaned
    .trim_start_matches("sec-")
    .trim_start_matches("ssec-")
    .trim_start_matches("opt-");
  let spaced = without_prefix.replace(['-', '_'], " ");
  spaced
    .split_whitespace()
    .map(|word| {
      let mut chars = word.chars();
      chars.next().map_or_else(String::new, |c| {
        c.to_uppercase().collect::<String>() + chars.as_str()
      })
    })
    .collect::<Vec<String>>()
    .join(" ")
}

/// Compute the relative URL from one output page to another.
///
/// E.g. `relative_page_path("guide/intro.html", "install.html")` →
/// `"../install.html"`.
fn relative_page_path(from_page: &str, to_page: &str) -> String {
  let from_dir = Path::new(from_page)
    .parent()
    .unwrap_or_else(|| Path::new(""));
  let to_path = Path::new(to_page);

  let from_parts: Vec<_> = from_dir
    .components()
    .filter(|c| !matches!(c, Component::CurDir))
    .collect();
  let to_parts: Vec<_> = to_path
    .components()
    .filter(|c| !matches!(c, Component::CurDir))
    .collect();

  let common = from_parts
    .iter()
    .zip(to_parts.iter())
    .take_while(|(a, b)| a == b)
    .count();

  let ups = from_parts.len() - common;
  let remainder = &to_parts[common..];

  let mut result = std::path::PathBuf::new();
  for _ in 0..ups {
    result.push("..");
  }
  for part in remainder {
    result.push(part);
  }

  let s = result.to_string_lossy().to_string();
  if s.is_empty() { to_page.to_string() } else { s }
}

/// Rewrite cross-page anchor links in rendered HTML.
///
/// For every `<a href="#anchor">` element:
/// - If `anchor` is found in `registry` and its owning page differs from
///   `current_page`, rewrites the `href` to the relative path to that page plus
///   the fragment (`other.html#anchor`).
/// - If the link text is empty, the `{{ANCHOR}}` placeholder, or the
///   auto-generated humanized slug, replaces it with the registry title.
/// - Same-page anchors and anchors absent from the registry are left as-is.
///
/// The `registry` maps anchor ID → `(owning_output_page, heading_title)`.
#[must_use]
#[expect(
  clippy::disallowed_types,
  reason = "Uses generic HashMap for hasher flexibility"
)]
pub fn rewrite_cross_page_anchor_links<S: std::hash::BuildHasher>(
  html: &str,
  current_page: &str,
  registry: &HashMap<String, (String, String), S>,
) -> String {
  if registry.is_empty() {
    return html.to_string();
  }

  kuchiki_postprocess_html(html, |document| {
    // Collect modifications first to avoid borrow conflicts.
    let mut modifications: Vec<(kuchikikiki::NodeRef, String, Option<String>)> =
      Vec::new();

    for link_node in safe_select(document, "a[href^='#']") {
      let Some(element) = link_node.as_element() else {
        continue;
      };

      let href = element
        .attributes
        .borrow()
        .get(local_name!("href"))
        .map(std::string::ToString::to_string);
      let Some(href_val) = href else { continue };

      let anchor_id = href_val.trim_start_matches('#');
      let Some((target_page, target_title)) = registry.get(anchor_id) else {
        continue;
      };

      if target_page == current_page {
        continue;
      }

      let rel = relative_page_path(current_page, target_page);
      let new_href = format!("{rel}#{anchor_id}");

      let current_text = link_node.text_contents();
      let humanized = humanize_anchor(&href_val);
      let replace_text = current_text.trim().is_empty()
        || current_text.trim() == "{{ANCHOR}}"
        || current_text.trim() == humanized.trim();

      let new_text = if replace_text {
        Some(target_title.clone())
      } else {
        None
      };

      modifications.push((link_node, new_href, new_text));
    }

    for (link_node, new_href, new_text) in modifications {
      if let Some(element) = link_node.as_element() {
        element
          .attributes
          .borrow_mut()
          .insert(local_name!("href"), new_href);
      }
      if let Some(text) = new_text {
        for child in link_node.children() {
          child.detach();
        }
        link_node.append(kuchikikiki::NodeRef::new_text(text));
      }
    }
  })
}

/// Extract all inline text from a heading node.
pub fn extract_inline_text<'a>(node: &'a AstNode<'a>) -> String {
  fn inner<'a>(node: &'a AstNode<'a>) -> String {
    let mut text = String::new();
    for child in node.children() {
      match &child.data.borrow().value {
        NodeValue::Text(t) => text.push_str(t),
        NodeValue::Code(t) => text.push_str(&t.literal),
        NodeValue::Link(..)
        | NodeValue::Emph
        | NodeValue::Strong
        | NodeValue::Strikethrough
        | NodeValue::Superscript
        | NodeValue::Subscript
        | NodeValue::FootnoteReference(..) => {
          text.push_str(&inner(child));
        },
        #[expect(clippy::match_same_arms, reason = "Explicit for clarity")]
        NodeValue::HtmlInline(_) | NodeValue::Image(..) => {},
        _ => {},
      }
    }
    text
  }
  inner(node)
}

/// Slugify heading text for use as a table-of-contents anchor.
///
/// The auto-generated heading `id` is produced by slugifying the *rendered*
/// HTML (see `process_header_anchors_html`), in which comrak has escaped any
/// markup characters: an option heading like `environments.<name>.deployment`
/// becomes `environments.&lt;name&gt;.deployment` before slugifying. The
/// table-of-contents, by contrast, slugifies the raw inline text (`<name>`),
/// which would yield a different slug and break "jump to header" for every
/// heading containing such characters.
///
/// To keep both in sync, escape the text the same way comrak does before
/// slugifying, so the TOC href matches the on-page heading `id`. The heading
/// `id` itself is intentionally left unchanged to preserve existing deep links.
#[must_use]
pub(crate) fn slugify_heading(text: &str) -> String {
  utils::slugify(&html_escape::encode_double_quoted_attribute(text))
}

/// Collect all markdown files from the input directory
pub fn collect_markdown_files(input_dir: &Path) -> Vec<PathBuf> {
  let mut files = Vec::with_capacity(100);

  for entry in WalkDir::new(input_dir)
    .follow_links(true)
    .into_iter()
    .filter_map(Result::ok)
  {
    let path = entry.path();
    if path.is_file() && path.extension().is_some_and(|ext| ext == "md") {
      files.push(path.to_owned());
    }
  }

  trace!("Found {} markdown files to process", files.len());
  files
}

/// Features that can be queried on a processor instance.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ProcessorFeature {
  /// GitHub Flavored Markdown support
  Gfm,
  /// Nixpkgs documentation extensions
  Nixpkgs,
  /// Syntax highlighting for code blocks
  SyntaxHighlighting,
  /// Manpage URL mapping support
  ManpageUrls,
}

fn admonition_lines(content: &str) -> Vec<bool> {
  let mut ignored = vec![false; content.lines().count() + 1];
  let mut depth = 0usize;

  for (index, line) in content.lines().enumerate() {
    let trimmed = line.trim_start();
    if trimmed.starts_with("<div class=\"admonition ") {
      depth += 1;
    }
    if depth > 0 {
      ignored[index + 1] = true;
    }
    if depth > 0 && trimmed == "</div>" {
      depth -= 1;
    }
  }

  ignored
}

fn is_setext_heading_underline(line: &str) -> bool {
  !line.is_empty()
    && (line.chars().all(|ch| ch == '=' || ch.is_whitespace())
      || line.chars().all(|ch| ch == '-' || ch.is_whitespace()))
}

/// Standalone HTML post-processing function to avoid borrowing issues.
fn kuchiki_postprocess_html<F>(html: &str, transform_fn: F) -> String
where
  F: FnOnce(&kuchikikiki::NodeRef),
{
  process_safe(
    html,
    |html| {
      use tendril::TendrilSink;

      let document = kuchikikiki::parse_html().one(html);
      transform_fn(&document);

      let mut out = Vec::new();
      let _ = document.serialize(&mut out);
      String::from_utf8_lossy(&out).into_owned()
    },
    html,
  )
}