use anyhow::{Context, Result};
use flate2::read::ZlibDecoder;
use flate2::write::ZlibEncoder;
use flate2::Compression;
use log::{debug, info};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::io::{Read, Write};
use std::path::Path;
use tokio::fs;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
pub struct InventoryItem {
pub project_name: String,
pub project_version: String,
pub uri: String,
pub display_name: String,
}
impl InventoryItem {
pub fn new(
project_name: String,
project_version: String,
uri: String,
display_name: String,
) -> Self {
Self {
project_name,
project_version,
uri,
display_name,
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Inventory {
pub data: BTreeMap<String, BTreeMap<String, InventoryItem>>,
}
impl Inventory {
pub fn new() -> Self {
Self {
data: BTreeMap::new(),
}
}
pub fn insert(&mut self, obj_type: String, name: String, item: InventoryItem) {
self.data.entry(obj_type).or_default().insert(name, item);
}
pub fn get(&self, obj_type: &str, name: &str) -> Option<&InventoryItem> {
self.data.get(obj_type)?.get(name)
}
pub fn contains(&self, obj_type: &str, name: &str) -> bool {
self.data
.get(obj_type)
.is_some_and(|objects| objects.contains_key(name))
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct InvObject {
pub name: String,
pub objtype: String,
pub priority: i32,
pub docname: String,
pub anchor: String,
pub dispname: String,
}
pub fn posix_join(uri: &str, location: &str) -> String {
if location.starts_with('/') {
location.to_string()
} else if uri.is_empty() || uri.ends_with('/') {
format!("{uri}{location}")
} else {
format!("{uri}/{location}")
}
}
lazy_static::lazy_static! {
static ref V2_LINE_RE: regex::Regex =
regex::Regex::new(r"^(.+?)\s+(\S+)\s+(-?\d+)\s+?(\S*)\s+(.*)").unwrap();
static ref WHITESPACE_RUN_RE: regex::Regex = regex::Regex::new(r"\s+").unwrap();
}
pub struct InventoryFile;
impl InventoryFile {
pub fn loads(content: &[u8], uri: &str) -> Result<Inventory> {
let (format_line, rest) = partition_bytes(content, b'\n');
let format_line = rstrip_bytes(format_line);
if format_line == b"# Sphinx inventory version 2" {
Self::loads_v2(rest, uri)
} else if format_line == b"# Sphinx inventory version 1" {
Self::loads_v1(rest, uri)
} else if let Some(unknown_version_bytes) =
format_line.strip_prefix(b"# Sphinx inventory version ")
{
let unknown_version = String::from_utf8(unknown_version_bytes.to_vec())
.context("inventory header version suffix is not valid UTF-8")?;
anyhow::bail!(
"unknown or unsupported inventory version: {}",
python_repr_str(&unknown_version)
);
} else {
let line = String::from_utf8(format_line.to_vec())
.context("inventory header line is not valid UTF-8")?;
anyhow::bail!("invalid inventory header: {}", line);
}
}
pub async fn load<P: AsRef<Path>>(filename: P, uri: &str) -> Result<Inventory> {
let content = fs::read(filename.as_ref()).await.with_context(|| {
format!(
"Failed to read inventory file: {}",
filename.as_ref().display()
)
})?;
Self::loads(&content, uri)
}
fn loads_v1(content: &[u8], uri: &str) -> Result<Inventory> {
let text =
String::from_utf8(content.to_vec()).context("v1 inventory body is not valid UTF-8")?;
let lines = python_str_splitlines(&text);
if lines.len() < 2 {
anyhow::bail!("invalid inventory header: missing project name or version");
}
let mut inv = Inventory::new();
let projname = str_slice_from_char(lines[0].trim_end(), 11).to_string();
let version = str_slice_from_char(lines[1].trim_end(), 11).to_string();
for line in &lines[2..] {
let fields = python_split_none_maxsplit(line.trim_end(), 2);
if fields.len() != 3 {
anyhow::bail!(
"invalid inventory v1 entry (expected `name type location`): {}",
line
);
}
let (name, item_type, location) = (fields[0], fields[1], fields[2]);
let mut location = posix_join(uri, location);
let domain_type = if item_type == "mod" {
location.push_str("#module-");
location.push_str(name);
"py:module".to_string()
} else {
location.push('#');
location.push_str(name);
format!("py:{item_type}")
};
let item =
InventoryItem::new(projname.clone(), version.clone(), location, "-".to_string());
inv.insert(domain_type, name.to_string(), item);
}
Ok(inv)
}
fn loads_v2(content: &[u8], uri: &str) -> Result<Inventory> {
let parts = splitn_bytes(content, b'\n', 4);
if parts.len() != 4 {
anyhow::bail!("invalid inventory header: missing project name or version");
}
let (line_1, line_2, check_line, compressed) = (parts[0], parts[1], parts[2], parts[3]);
let projname = String::from_utf8(bytes_slice_from(rstrip_bytes(line_1), 11).to_vec())
.context("inventory Project header is not valid UTF-8")?;
let version = String::from_utf8(bytes_slice_from(rstrip_bytes(line_2), 11).to_vec())
.context("inventory Version header is not valid UTF-8")?;
if !contains_bytes(check_line, b"zlib") {
let check_line_text = String::from_utf8(check_line.to_vec())
.context("inventory compression-check line is not valid UTF-8")?;
anyhow::bail!(
"invalid inventory header (not compressed): {}",
check_line_text
);
}
let decompressed = decompress_zlib(compressed)?;
let decompressed_text = String::from_utf8(decompressed)
.context("decompressed inventory payload is not valid UTF-8")?;
let mut inv = Inventory::new();
let mut potential_ambiguities: HashMap<String, (String, String, String)> = HashMap::new();
let mut actual_ambiguities: HashSet<String> = HashSet::new();
for line in python_str_splitlines(&decompressed_text) {
let trimmed = line.trim_end();
let Some(caps) = V2_LINE_RE.captures(trimmed) else {
continue;
};
let name = caps.get(1).unwrap().as_str();
let type_ = caps.get(2).unwrap().as_str();
let prio = caps.get(3).unwrap().as_str();
let mut location = caps.get(4).unwrap().as_str().to_string();
let dispname = caps.get(5).unwrap().as_str().to_string();
if !type_.contains(':') {
continue;
}
if type_ == "py:module" && inv.contains(type_, name) {
continue;
}
if type_ == "std:label" || type_ == "std:term" {
let definition = format!("{type_}:{name}");
let content_key = (prio.to_string(), location.clone(), dispname.clone());
let lowercase_definition = definition.to_lowercase();
match potential_ambiguities.get(&lowercase_definition) {
Some(existing) if existing == &content_key => {
debug!(
"inventory <{}> contains duplicate definitions of {}",
uri, definition
);
}
Some(_) => {
actual_ambiguities.insert(definition);
}
None => {
potential_ambiguities.insert(lowercase_definition, content_key);
}
}
}
if let Some(prefix) = location.strip_suffix('$') {
location = format!("{prefix}{name}");
}
let joined = posix_join(uri, &location);
let item = InventoryItem::new(projname.clone(), version.clone(), joined, dispname);
inv.insert(type_.to_string(), name.to_string(), item);
}
for ambiguity in &actual_ambiguities {
info!(
"inventory <{}> contains multiple definitions for {}",
uri, ambiguity
);
}
Ok(inv)
}
pub async fn dump<P: AsRef<Path>>(
path: P,
project: &str,
version: &str,
domains: &[(&str, Vec<InvObject>)],
get_target_uri: impl Fn(&str) -> String,
) -> Result<()> {
let header = format!(
"# Sphinx inventory version 2\n\
# Project: {}\n\
# Version: {}\n\
# The remainder of this file is compressed using zlib.\n",
Self::escape_string(project),
Self::escape_string(version),
);
let mut sorted_domains: Vec<&(&str, Vec<InvObject>)> = domains.iter().collect();
sorted_domains.sort_by_key(|(name, _)| *name);
let mut body = Vec::new();
for (domain_name, objects) in sorted_domains {
let mut objects: Vec<&InvObject> = objects.iter().collect();
objects.sort_by(|a, b| {
a.name
.cmp(&b.name)
.then_with(|| a.dispname.cmp(&b.dispname))
.then_with(|| a.objtype.cmp(&b.objtype))
.then_with(|| a.docname.cmp(&b.docname))
.then_with(|| a.anchor.cmp(&b.anchor))
.then_with(|| a.priority.cmp(&b.priority))
});
for obj in objects {
let anchor = match obj.anchor.strip_suffix(obj.name.as_str()) {
Some(prefix) => format!("{prefix}$"),
None => obj.anchor.clone(),
};
let mut uri = get_target_uri(&obj.docname);
if !anchor.is_empty() {
uri.push('#');
uri.push_str(&anchor);
}
let dispname: &str = if obj.dispname == obj.name {
"-"
} else {
obj.dispname.as_str()
};
let line = format!(
"{} {}:{} {} {} {}\n",
obj.name, domain_name, obj.objtype, obj.priority, uri, dispname
);
body.extend_from_slice(line.as_bytes());
}
}
let mut encoder = ZlibEncoder::new(Vec::new(), Compression::new(9));
encoder
.write_all(&body)
.context("failed to compress inventory body")?;
let compressed = encoder
.finish()
.context("failed to finalize inventory zlib stream")?;
let mut content = header.into_bytes();
content.extend_from_slice(&compressed);
fs::write(path, content)
.await
.context("Failed to write inventory file")?;
Ok(())
}
fn escape_string(s: &str) -> String {
WHITESPACE_RUN_RE.replace_all(s, " ").to_string()
}
}
fn partition_bytes(data: &[u8], sep: u8) -> (&[u8], &[u8]) {
match data.iter().position(|&b| b == sep) {
Some(pos) => (&data[..pos], &data[pos + 1..]),
None => (data, &[]),
}
}
fn splitn_bytes(data: &[u8], sep: u8, n: usize) -> Vec<&[u8]> {
let mut parts = Vec::with_capacity(n);
let mut rest = data;
while parts.len() + 1 < n {
match rest.iter().position(|&b| b == sep) {
Some(pos) => {
parts.push(&rest[..pos]);
rest = &rest[pos + 1..];
}
None => break,
}
}
parts.push(rest);
parts
}
fn rstrip_bytes(data: &[u8]) -> &[u8] {
let mut end = data.len();
while end > 0 && matches!(data[end - 1], b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c) {
end -= 1;
}
&data[..end]
}
fn bytes_slice_from(data: &[u8], start: usize) -> &[u8] {
if start >= data.len() {
&[]
} else {
&data[start..]
}
}
fn str_slice_from_char(s: &str, start: usize) -> &str {
match s.char_indices().nth(start) {
Some((byte_idx, _)) => &s[byte_idx..],
None => "",
}
}
fn contains_bytes(data: &[u8], needle: &[u8]) -> bool {
if needle.is_empty() {
return true;
}
data.windows(needle.len()).any(|w| w == needle)
}
fn python_str_splitlines(s: &str) -> Vec<&str> {
let mut lines = Vec::new();
let mut start = 0usize;
let mut chars = s.char_indices().peekable();
while let Some((idx, ch)) = chars.next() {
let is_boundary = matches!(
ch,
'\n' | '\r'
| '\u{0b}'
| '\u{0c}'
| '\u{1c}'
| '\u{1d}'
| '\u{1e}'
| '\u{85}'
| '\u{2028}'
| '\u{2029}'
);
if is_boundary {
lines.push(&s[start..idx]);
let mut end = idx + ch.len_utf8();
if ch == '\r' {
if let Some(&(_, '\n')) = chars.peek() {
let (nidx, nch) = chars.next().unwrap();
end = nidx + nch.len_utf8();
}
}
start = end;
}
}
if start < s.len() {
lines.push(&s[start..]);
}
lines
}
fn python_split_none_maxsplit(s: &str, maxsplit: usize) -> Vec<&str> {
let mut result = Vec::new();
let mut rest = s;
loop {
let trimmed = rest.trim_start();
if trimmed.is_empty() {
break;
}
if result.len() == maxsplit {
result.push(trimmed);
break;
}
match trimmed.find(char::is_whitespace) {
Some(idx) => {
result.push(&trimmed[..idx]);
rest = &trimmed[idx..];
}
None => {
result.push(trimmed);
rest = "";
}
}
}
result
}
fn python_repr_str(s: &str) -> String {
let has_single = s.contains('\'');
let has_double = s.contains('"');
let quote = if has_single && !has_double { '"' } else { '\'' };
let mut out = String::with_capacity(s.len() + 2);
out.push(quote);
for c in s.chars() {
match c {
'\\' => out.push_str("\\\\"),
c if c == quote => {
out.push('\\');
out.push(c);
}
'\n' => out.push_str("\\n"),
'\r' => out.push_str("\\r"),
'\t' => out.push_str("\\t"),
c if (c as u32) < 0x20 || (c as u32) == 0x7f => {
out.push_str(&format!("\\x{:02x}", c as u32));
}
c => out.push(c),
}
}
out.push(quote);
out
}
fn decompress_zlib(data: &[u8]) -> Result<Vec<u8>> {
let mut decoder = ZlibDecoder::new(data);
let mut decompressed = Vec::new();
decoder
.read_to_end(&mut decompressed)
.context("failed to decompress inventory zlib payload")?;
Ok(decompressed)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_inventory_item_creation() {
let item = InventoryItem::new(
"test_project".to_string(),
"1.0".to_string(),
"http://example.com/test.html".to_string(),
"Test Item".to_string(),
);
assert_eq!(item.project_name, "test_project");
assert_eq!(item.project_version, "1.0");
assert_eq!(item.uri, "http://example.com/test.html");
assert_eq!(item.display_name, "Test Item");
}
#[test]
fn test_inventory_operations() {
let mut inv = Inventory::new();
let item = InventoryItem::new(
"test".to_string(),
"1.0".to_string(),
"test.html".to_string(),
"Test".to_string(),
);
inv.insert(
"py:function".to_string(),
"test_func".to_string(),
item.clone(),
);
assert!(inv.contains("py:function", "test_func"));
assert_eq!(inv.get("py:function", "test_func"), Some(&item));
assert!(!inv.contains("py:function", "nonexistent"));
}
#[test]
fn test_escape_string() {
assert_eq!(
InventoryFile::escape_string("test multiple spaces"),
"test multiple spaces"
);
assert_eq!(InventoryFile::escape_string("test\ttab"), "test tab");
assert_eq!(
InventoryFile::escape_string("test\nnewline"),
"test newline"
);
}
#[test]
fn test_posix_join_inserts_separator() {
assert_eq!(posix_join("/util", "foo.html"), "/util/foo.html");
}
#[test]
fn test_posix_join_no_double_separator() {
assert_eq!(posix_join("/util/", "foo.html"), "/util/foo.html");
}
#[test]
fn test_posix_join_empty_location() {
assert_eq!(posix_join("/util", ""), "/util/");
}
#[test]
fn test_posix_join_empty_uri() {
assert_eq!(posix_join("", "foo.html"), "foo.html");
}
#[test]
fn test_posix_join_absolute_location_overrides_uri() {
assert_eq!(posix_join("/util", "/abs/path.html"), "/abs/path.html");
}
#[test]
fn test_posix_join_both_empty() {
assert_eq!(posix_join("", ""), "");
}
#[test]
fn test_posix_join_uri_with_scheme() {
assert_eq!(
posix_join("https://example.org/v1", "sub/x.html#y"),
"https://example.org/v1/sub/x.html#y"
);
}
#[test]
fn test_splitlines_mixed_separators() {
assert_eq!(
python_str_splitlines("a\r\nb\rc\u{0b}d\u{0c}e"),
vec!["a", "b", "c", "d", "e"]
);
}
#[test]
fn test_splitlines_no_trailing_empty() {
assert_eq!(python_str_splitlines("a\nb\n"), vec!["a", "b"]);
}
#[test]
fn test_splitlines_empty_string() {
assert!(python_str_splitlines("").is_empty());
}
#[test]
fn test_splitlines_lone_newline() {
assert_eq!(python_str_splitlines("\n"), vec![""]);
}
#[test]
fn test_splitlines_embedded_blank_line() {
assert_eq!(python_str_splitlines("a\n\nb"), vec!["a", "", "b"]);
}
#[test]
fn test_split_none_maxsplit_collapses_runs() {
assert_eq!(
python_split_none_maxsplit("module mod foo.html", 2),
vec!["module", "mod", "foo.html"]
);
}
#[test]
fn test_split_none_maxsplit_remainder_keeps_internal_whitespace() {
assert_eq!(
python_split_none_maxsplit("a b c d e", 2),
vec!["a", "b", "c d e"]
);
}
#[test]
fn test_split_none_maxsplit_empty() {
assert!(python_split_none_maxsplit("", 2).is_empty());
assert!(python_split_none_maxsplit(" ", 2).is_empty());
}
#[test]
fn test_split_none_maxsplit_too_few_tokens() {
assert_eq!(python_split_none_maxsplit("onlyone", 2), vec!["onlyone"]);
}
#[test]
fn test_python_repr_str_plain() {
assert_eq!(python_repr_str("5"), "'5'");
}
#[test]
fn test_python_repr_str_prefers_single_quotes() {
assert_eq!(python_repr_str("2.5-beta"), "'2.5-beta'");
}
#[test]
fn test_python_repr_str_switches_to_double_quotes() {
assert_eq!(python_repr_str("it's"), "\"it's\"");
}
#[test]
fn test_v2_line_regex_no_match_on_garbage() {
assert!(V2_LINE_RE
.captures("not a valid entry line at all")
.is_none());
}
#[test]
fn test_v2_line_regex_captures_five_groups() {
let caps = V2_LINE_RE
.captures("a term including:colon std:term -1 glossary.html#term -")
.unwrap();
assert_eq!(&caps[1], "a term including:colon");
assert_eq!(&caps[2], "std:term");
assert_eq!(&caps[3], "-1");
assert_eq!(&caps[4], "glossary.html#term");
assert_eq!(&caps[5], "-");
}
}