use std::collections::{HashMap, HashSet};
use unicode_normalization::UnicodeNormalization;
use super::messages;
use super::Node;
use crate::utils::py_isspace;
fn translate_digraph(c: char) -> Option<&'static str> {
Some(match c as u32 {
223 => "sz", 230 => "ae", 339 => "oe", 568 => "db", 569 => "qp", _ => return None,
})
}
fn translate_single(c: char) -> Option<char> {
Some(match c as u32 {
248 => 'o', 273 => 'd', 295 => 'h', 305 => 'i', 322 => 'l', 359 => 't', 384 => 'b', 387 => 'b', 392 => 'c', 396 => 'd', 402 => 'f', 409 => 'k', 410 => 'l', 414 => 'n', 421 => 'p', 427 => 't', 429 => 't', 436 => 'y', 438 => 'z', 485 => 'g', 549 => 'z', 564 => 'l', 565 => 'n', 566 => 't', 567 => 'j', 572 => 'c', 575 => 's', 576 => 'z', 583 => 'e', 585 => 'j', 587 => 'q', 589 => 'r', 591 => 'y', _ => return None,
})
}
pub fn make_id(s: &str) -> String {
let lowered = s.to_lowercase();
let mut translated = String::with_capacity(lowered.len());
for c in lowered.chars() {
if let Some(d) = translate_digraph(c) {
translated.push_str(d);
} else if let Some(r) = translate_single(c) {
translated.push(r);
} else {
translated.push(c);
}
}
let ascii: String = translated.nfkd().filter(char::is_ascii).collect();
let collapsed = py_split_join(&ascii);
let mut out = String::with_capacity(collapsed.len());
let mut in_run = false;
for c in collapsed.chars() {
if c.is_ascii_lowercase() || c.is_ascii_digit() {
out.push(c);
in_run = false;
} else if !in_run {
out.push('-');
in_run = true;
}
}
let bytes = out.as_bytes();
let mut start = 0;
while start < bytes.len() && (bytes[start] == b'-' || bytes[start].is_ascii_digit()) {
start += 1;
}
let mut end = bytes.len();
while end > start && bytes[end - 1] == b'-' {
end -= 1;
}
out[start..end].to_string()
}
pub fn sphinx_make_id(s: &str) -> String {
let mut translated = String::with_capacity(s.len());
for c in s.chars() {
if let Some(d) = translate_digraph(c) {
translated.push_str(d);
} else if let Some(r) = translate_single(c) {
translated.push(r);
} else {
translated.push(c);
}
}
let ascii: String = translated.nfkd().filter(char::is_ascii).collect();
let collapsed = py_split_join(&ascii);
let mut out = String::with_capacity(collapsed.len());
let mut in_run = false;
for c in collapsed.chars() {
if c.is_ascii_alphanumeric() || c == '.' || c == '_' {
out.push(c);
in_run = false;
} else if !in_run {
out.push('-');
in_run = true;
}
}
let bytes = out.as_bytes();
let mut start = 0;
while start < bytes.len() && matches!(bytes[start], b'-' | b'.' | b'_' | b'0'..=b'9') {
start += 1;
}
let mut end = bytes.len();
while end > start && bytes[end - 1] == b'-' {
end -= 1;
}
out[start..end].to_string()
}
fn py_split_join(s: &str) -> String {
s.split(py_isspace)
.filter(|w| !w.is_empty())
.collect::<Vec<_>>()
.join(" ")
}
pub fn fully_normalize_name(s: &str) -> String {
py_split_join(&s.to_lowercase())
}
pub fn whitespace_normalize_name(s: &str) -> String {
py_split_join(s)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DupnameFixup {
pub name: String,
pub node_id: String,
}
#[derive(Debug, Clone)]
struct NameEntry {
id: Option<String>,
explicit: bool,
refuri: Option<String>,
}
#[derive(Debug, Default)]
pub struct IdRegistry {
ids: HashSet<String>,
nameids: HashMap<String, NameEntry>,
id_counter: HashMap<&'static str, u64>,
fixups: Vec<DupnameFixup>,
index_serial: u32,
serialnos: HashMap<String, u32>,
}
impl IdRegistry {
pub fn new() -> Self {
Self::default()
}
pub fn new_index_serialno(&mut self) -> u32 {
let n = self.index_serial;
self.index_serial += 1;
n
}
pub fn sphinx_make_id(&mut self, prefix: &str, term: &str) -> String {
let mut node_id = if term.is_empty() {
None
} else if prefix.is_empty() {
let candidate = sphinx_make_id(term);
(!candidate.is_empty()).then_some(candidate)
} else {
let candidate = sphinx_make_id(&format!("{prefix}-{term}"));
(candidate != prefix).then_some(candidate)
};
loop {
match &node_id {
Some(id) if !self.ids.contains(id) => return id.clone(),
_ => {}
}
let counter = self.serialnos.entry(prefix.to_string()).or_insert(0);
let serial = *counter;
*counter += 1;
node_id = Some(if prefix.is_empty() {
format!("id{serial}")
} else {
format!("{prefix}-{serial}")
});
}
}
pub fn note_explicit_id(&mut self, id: &str) {
self.ids.insert(id.to_string());
}
fn allocate_id(&mut self, names: &[String]) -> String {
for name in names {
let base = make_id(name);
if !base.is_empty() && !self.ids.contains(&base) {
self.ids.insert(base.clone());
return base;
}
}
loop {
let counter = self.id_counter.entry("id").or_insert(0);
*counter += 1;
let id = format!("id{counter}");
if !self.ids.contains(&id) {
self.ids.insert(id.clone());
return id;
}
}
}
fn dupname_new(node: &mut Node, name: &str) {
if let Some(pos) = node.attrs.names.iter().position(|n| n == name) {
node.attrs.names.remove(pos);
node.attrs.dupnames.push(name.to_string());
}
}
fn register(
&mut self,
node: &mut Node,
line: u32,
source: &str,
explicit: bool,
backrefs_on_msg: bool,
refuri: Option<&str>,
) -> Option<Node> {
let id = self.allocate_id(&node.attrs.names);
node.attrs.ids.push(id.clone());
let mut message = None;
let names = node.attrs.names.clone();
for name in names {
let Some(entry) = self.nameids.get(&name).cloned() else {
self.nameids.insert(
name,
NameEntry {
id: Some(id.clone()),
explicit,
refuri: refuri.map(str::to_string),
},
);
continue;
};
let dup_info = |level: u8, text: String, with_backrefs: bool| {
let mut msg = messages::system_message(level, &text, line, source);
if with_backrefs {
msg.attrs.backrefs.push(id.clone());
}
msg
};
match (entry.explicit, explicit) {
(true, true) => {
if refuri.is_some() && entry.refuri.as_deref() == refuri {
Self::dupname_new(node, &name);
continue;
}
if let Some(old_id) = entry.id.clone() {
self.fixups.push(DupnameFixup {
name: name.clone(),
node_id: old_id,
});
}
self.nameids.insert(
name.clone(),
NameEntry {
id: None,
explicit: true,
refuri: None,
},
);
Self::dupname_new(node, &name);
message = Some(dup_info(
messages::WARNING,
format!("Duplicate explicit target name: \"{name}\"."),
backrefs_on_msg,
));
}
(true, false) => {
Self::dupname_new(node, &name);
message = Some(dup_info(
messages::INFO,
format!("Duplicate implicit target name: \"{name}\"."),
backrefs_on_msg,
));
}
(false, true) => {
if let Some(old_id) = entry.id.clone() {
self.fixups.push(DupnameFixup {
name: name.clone(),
node_id: old_id,
});
}
self.nameids.insert(
name.clone(),
NameEntry {
id: Some(id.clone()),
explicit: true,
refuri: refuri.map(str::to_string),
},
);
message = Some(dup_info(
messages::INFO,
format!("Target name overrides implicit target name \"{name}\"."),
false,
));
}
(false, false) => {
if let Some(old_id) = entry.id.clone() {
self.fixups.push(DupnameFixup {
name: name.clone(),
node_id: old_id,
});
}
self.nameids.insert(
name.clone(),
NameEntry {
id: None,
explicit: false,
refuri: None,
},
);
Self::dupname_new(node, &name);
message = Some(dup_info(
messages::INFO,
format!("Duplicate implicit target name: \"{name}\"."),
backrefs_on_msg,
));
}
}
}
message
}
pub fn set_id_implicit(&mut self, node: &mut Node, line: u32, source: &str) -> Option<Node> {
self.register(node, line, source, false, true, None)
}
pub fn set_id_explicit(
&mut self,
node: &mut Node,
line: u32,
source: &str,
internal: bool,
refuri: Option<&str>,
) -> Option<Node> {
self.register(node, line, source, true, internal, refuri)
}
pub fn set_id_anonymous(&mut self, node: &mut Node) {
let id = self.allocate_id(&[]);
node.attrs.ids.push(id);
}
pub fn allocate_auto_id(&mut self) -> String {
self.allocate_id(&[])
}
pub fn take_fixups(&mut self) -> Vec<DupnameFixup> {
std::mem::take(&mut self.fixups)
}
pub fn nameids_snapshot(&self) -> Vec<(String, Option<String>, bool)> {
self.nameids
.iter()
.map(|(name, entry)| (name.clone(), entry.id.clone(), entry.explicit))
.collect()
}
pub fn index_serial(&self) -> u32 {
self.index_serial
}
}
pub fn apply_dupname_fixups(root: &mut Node, fixups: &[DupnameFixup]) {
if fixups.is_empty() {
return;
}
for fixup in fixups {
apply_one_fixup(root, fixup);
}
}
fn apply_one_fixup(node: &mut Node, fixup: &DupnameFixup) -> bool {
if node.attrs.ids.contains(&fixup.node_id) {
if let Some(pos) = node.attrs.names.iter().position(|n| *n == fixup.name) {
node.attrs.names.remove(pos);
node.attrs.dupnames.push(fixup.name.clone());
}
return true;
}
node.children
.iter_mut()
.any(|child| apply_one_fixup(child, fixup))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::doctree::{kinds, AttrValue, Node, Span};
#[test]
fn name_normalizers_split_on_python_whitespace() {
assert_eq!(fully_normalize_name("a\x1fb"), "a b");
assert_eq!(fully_normalize_name("Sec\x1fC"), "sec c");
assert_eq!(fully_normalize_name("\x1f a \x1c\x1d\x1e b \x1f"), "a b");
assert_eq!(whitespace_normalize_name("A\x1fB"), "A B");
assert_eq!(whitespace_normalize_name("\x1fA B\x1f"), "A B");
assert_eq!(make_id("a\x1fb"), "a-b");
assert_eq!(make_id("\x1fa\x1f"), "a");
assert_eq!(sphinx_make_id("envvar-FOO\x1fBAR"), "envvar-FOO-BAR");
assert_eq!(sphinx_make_id("\x1fA.b\x1f"), "A.b");
}
#[test]
fn make_id_basics() {
assert_eq!(make_id("My Section Title!"), "my-section-title");
assert_eq!(make_id("Hello World!"), "hello-world");
assert_eq!(make_id("1. Intro"), "intro");
assert_eq!(make_id("2026 report"), "report");
assert_eq!(make_id("Überblick"), "uberblick");
assert_eq!(make_id("straße"), "strasze");
assert_eq!(make_id("!!!"), "");
assert_eq!(make_id("123"), "");
assert_eq!(make_id("..."), "");
}
#[test]
fn sphinx_make_id_keeps_case_dots_and_underscores() {
assert_eq!(sphinx_make_id("envvar-HOME_A"), "envvar-HOME_A");
assert_eq!(sphinx_make_id("confval-my_setting"), "confval-my_setting");
assert_eq!(sphinx_make_id("a.b.C"), "a.b.C");
assert_eq!(
sphinx_make_id("cmdoption-myprog---verbose"),
"cmdoption-myprog-verbose"
);
assert_eq!(
sphinx_make_id("term-source directory"),
"term-source-directory"
);
assert_eq!(sphinx_make_id("._-1abc--"), "abc");
assert_eq!(sphinx_make_id("Überblick"), "Uberblick");
assert_eq!(sphinx_make_id("!!!"), "");
}
#[test]
fn sphinx_registry_make_id_falls_back_to_a_per_prefix_serial() {
let mut reg = IdRegistry::new();
assert_eq!(reg.sphinx_make_id("envvar", "HOME"), "envvar-HOME");
assert_eq!(reg.sphinx_make_id("envvar", "HOME"), "envvar-HOME");
reg.note_explicit_id("envvar-HOME");
assert_eq!(reg.sphinx_make_id("envvar", "HOME"), "envvar-0");
reg.note_explicit_id("envvar-0");
assert_eq!(reg.sphinx_make_id("envvar", "HOME"), "envvar-1");
assert_eq!(reg.sphinx_make_id("cmdoption", "!!!"), "cmdoption-0");
}
#[test]
fn sphinx_registry_make_id_empty_prefix_keeps_fullname_and_serials_as_id_n() {
let mut reg = IdRegistry::new();
assert_eq!(reg.sphinx_make_id("", "mymod.C.meth"), "mymod.C.meth");
reg.note_explicit_id("mymod.C.meth");
assert_eq!(reg.sphinx_make_id("", "mymod.C.meth"), "id0");
reg.note_explicit_id("id0");
assert_eq!(reg.sphinx_make_id("", "mymod.C.meth"), "id1");
reg.note_explicit_id("id1");
assert_eq!(reg.sphinx_make_id("", "!!!"), "id2");
reg.note_explicit_id("id2");
assert_eq!(reg.sphinx_make_id("", ""), "id3");
}
#[test]
fn name_normalization() {
assert_eq!(
fully_normalize_name("My Phrase Target"),
"my phrase target"
);
assert_eq!(fully_normalize_name("Hello World!"), "hello world!");
assert_eq!(fully_normalize_name("Überblick"), "überblick");
assert_eq!(whitespace_normalize_name("A B"), "A B");
}
#[test]
fn registry_assigns_ids_and_handles_implicit_duplicates() {
let mut reg = IdRegistry::new();
let mut s1 = Node::elem(kinds::SECTION, Span::ZERO);
s1.attrs.names.push("duplicate".into());
assert!(reg.set_id_implicit(&mut s1, 3, "<snippet>").is_none());
assert_eq!(s1.attrs.ids, vec!["duplicate"]);
let mut s2 = Node::elem(kinds::SECTION, Span::ZERO);
s2.attrs.names.push("duplicate".into());
let msg = reg
.set_id_implicit(&mut s2, 7, "<snippet>")
.expect("dup INFO");
assert_eq!(s2.attrs.ids, vec!["id1"]);
assert!(s2.attrs.names.is_empty());
assert_eq!(s2.attrs.dupnames, vec!["duplicate"]);
assert_eq!(msg.get("type"), Some(&AttrValue::Str("INFO".into())));
assert_eq!(msg.get("line"), Some(&AttrValue::Int(7)));
assert_eq!(msg.attrs.backrefs, vec!["id1"]);
let fixups = reg.take_fixups();
assert_eq!(
fixups,
vec![DupnameFixup {
name: "duplicate".into(),
node_id: "duplicate".into()
}]
);
let mut root = Node::elem(kinds::DOCUMENT, Span::ZERO);
root.children.push(s1);
apply_dupname_fixups(&mut root, &fixups);
let s1 = &root.children[0];
assert!(s1.attrs.names.is_empty());
assert_eq!(s1.attrs.dupnames, vec!["duplicate"]);
assert_eq!(s1.attrs.ids, vec!["duplicate"]); }
#[test]
fn registry_auto_ids_for_unmakeable_names() {
let mut reg = IdRegistry::new();
for (i, title) in ["!!!", "123", "..."].iter().enumerate() {
let mut s = Node::elem(kinds::SECTION, Span::ZERO);
s.attrs.names.push(fully_normalize_name(title));
reg.set_id_implicit(&mut s, 1, "<snippet>");
assert_eq!(s.attrs.ids, vec![format!("id{}", i + 1)]);
assert_eq!(s.attrs.names.len(), 1); }
}
#[test]
fn explicit_duplicate_warning_backrefs_only_when_internal() {
let mut reg = IdRegistry::new();
let mut t1 = Node::elem(kinds::TARGET, Span::ZERO);
t1.attrs.names.push("dup".into());
assert!(reg
.set_id_explicit(&mut t1, 1, "<snippet>", false, Some("https://1/"))
.is_none());
let mut t2 = Node::elem(kinds::TARGET, Span::ZERO);
t2.attrs.names.push("dup".into());
let msg = reg
.set_id_explicit(&mut t2, 3, "<snippet>", false, Some("https://2/"))
.expect("dup WARNING");
assert_eq!(msg.get("type"), Some(&AttrValue::Str("WARNING".into())));
assert!(msg.attrs.backrefs.is_empty()); assert_eq!(t2.attrs.ids, vec!["id1"]);
assert_eq!(t2.attrs.dupnames, vec!["dup"]);
let mut reg = IdRegistry::new();
let mut i1 = Node::elem(kinds::TARGET, Span::ZERO);
i1.attrs.names.push("t".into());
reg.set_id_explicit(&mut i1, 1, "<snippet>", true, None);
let mut i2 = Node::elem(kinds::TARGET, Span::ZERO);
i2.attrs.names.push("t".into());
let msg = reg
.set_id_explicit(&mut i2, 5, "<snippet>", true, None)
.expect("dup WARNING");
assert_eq!(msg.attrs.backrefs, vec!["id1"]); }
}