cargo_doc2readme/

output.rs

use crate::{
	diagnostic,
	input::{InputFile, Scope, TargetType},
	links::Links
};
use itertools::Itertools as _;
use miette::{Context as _, IntoDiagnostic as _};
use pulldown_cmark::{
	BrokenLink, CodeBlockKind, CowStr, Event, HeadingLevel, LinkType, Options, Parser,
	Tag, TagEnd
};
use semver::Version;
use serde::Serialize;
use std::{collections::BTreeMap, fmt::Write as _, io};
use syn::Path;
use url::Url;

const DEFAULT_CODEBLOCK_LANG: &str = "rust";
/// List of codeblock flags that rustdoc allows
const RUSTDOC_CODEBLOCK_FLAGS: &[&str] = &[
	"compile_fail",
	"edition2015",
	"edition2018",
	"edition2021",
	"edition2024",
	"ignore",
	"no_run",
	"should_panic"
];
const RUSTDOC_CODEBLOCK_PREFIXES: &[&str] = &["ignore-"];

pub struct ResolvedLink {
	pub path: String,
	pub link_type: Option<crate::input::LinkType>
}

impl Scope {
	pub fn resolve(&self, crate_name: &str, path: String) -> ResolvedLink {
		self.resolve_impl(crate_name, None, path)
	}

	pub fn resolve_impl(
		&self,
		crate_name: &str,
		link_type: Option<crate::input::LinkType>,
		path: String
	) -> ResolvedLink {
		if !path.starts_with("::") {
			// split path into segments, ignoring <...> generics
			let mut path = path.clone();
			loop {
				let idx = match (path.find('<'), path.rfind('>')) {
					(Some(idx1), Some(idx2)) if idx1 < idx2 => idx1,
					_ => break
				};
				let mut end = idx + 1;
				let mut depth: usize = 1;
				for ch in path[end ..].chars() {
					if ch == '<' {
						depth += 1;
					} else if ch == '>' {
						depth -= 1;
					}
					end += ch.len_utf8();

					if depth == 0 {
						break;
					}
				}
				path.replace_range(idx .. end, "");
			}
			// debug!("Trying to resolve path {path:?}");
			let mut segments = path.split("::").collect::<Vec<_>>();
			if segments[0] == "crate" {
				segments[0] = crate_name;
			}

			// check if we can resolve anything
			if self.scope.contains_key(segments[0]) {
				let paths = &self.scope[segments[0]];
				if let Some((path0_link_type, path0)) = paths.front() {
					segments[0] = path0;
					let resolved_path = segments.join("::");
					// debug!("Resolved path {path:?} to {resolved_path:?} as child of {path0:?} ({path0_link_type:?})");
					if path0.starts_with("::") {
						return ResolvedLink {
							path: resolved_path,
							link_type: if segments.len() == 1 {
								Some(*path0_link_type)
							} else {
								link_type
							}
						};
					}
					return self.resolve(crate_name, resolved_path);
				}
			}
		}

		ResolvedLink { path, link_type }
	}
}

fn broken_link_callback<'a>(lnk: BrokenLink<'_>) -> Option<(CowStr<'a>, CowStr<'a>)> {
	Some(("".into(), lnk.reference.to_string().into()))
}

fn filter_hidden_rust_codeblock_lines(line: &str) -> Option<&str> {
	match line.strip_prefix('#') {
		Some(stripped_line) => match stripped_line.chars().next() {
			// ignore "#" lines
			None => None,
			// replace "##" at the start of a line with "#"
			Some('#') => Some(stripped_line),
			// ignore lines starting with "#" if it is followed by whitespace
			Some(c) if c.is_whitespace() => None,
			// don't ignore the line if the "#" isn't followed by whitespace
			Some(_) => Some(line)
		},
		None => Some(line)
	}
}

struct EventFilter<'a, I: Iterator<Item = Event<'a>>> {
	iter: I,
	links: &'a mut BTreeMap<String, String>,

	in_code_block: bool,
	in_rust_code_block: bool,
	link_idx: usize
}

impl<'a, I: Iterator<Item = Event<'a>>> EventFilter<'a, I> {
	fn new(iter: I, links: &'a mut BTreeMap<String, String>) -> Self {
		Self {
			iter,
			links,

			in_code_block: false,
			in_rust_code_block: false,
			link_idx: 0
		}
	}
}

impl<'a, I: Iterator<Item = Event<'a>>> Iterator for EventFilter<'a, I> {
	type Item = Event<'a>;

	fn next(&mut self) -> Option<Self::Item> {
		loop {
			break Some(match self.iter.next()? {
				Event::Start(tag) => Event::Start(match tag {
					// we increase headings by 1 level
					Tag::Heading {
						level,
						id,
						classes,
						attrs
					} => {
						let level = match level {
							HeadingLevel::H1 => HeadingLevel::H2,
							HeadingLevel::H2 => HeadingLevel::H3,
							HeadingLevel::H3 => HeadingLevel::H4,
							HeadingLevel::H4 => HeadingLevel::H5,
							_ => HeadingLevel::H6
						};
						Tag::Heading {
							level,
							id,
							classes,
							attrs
						}
					},

					// we record codeblocks and adjust their language
					Tag::CodeBlock(kind) => {
						debug_assert!(
							!self.in_code_block,
							"Recursive codeblocks, wtf???"
						);
						self.in_code_block = true;
						Tag::CodeBlock(CodeBlockKind::Fenced(match kind {
							CodeBlockKind::Indented => DEFAULT_CODEBLOCK_LANG.into(),
							CodeBlockKind::Fenced(lang) => {
								let mut lang: String = (*lang).to_owned();
								for prefix in RUSTDOC_CODEBLOCK_PREFIXES {
									while let Some(idx_start) = lang.find(prefix) {
										let idx_off = idx_start + prefix.len();
										match lang[idx_off ..].find(',') {
											Some(idx_end) => lang.replace_range(
												idx_start ..= idx_off + idx_end,
												""
											),
											None => lang.replace_range(idx_start .., "")
										}
									}
								}
								for flag in RUSTDOC_CODEBLOCK_FLAGS {
									lang = lang.replace(flag, "");
								}
								let mut lang: CowStr<'_> = lang.replace(',', "").into();
								if lang.is_empty() {
									lang = DEFAULT_CODEBLOCK_LANG.into();
								}
								self.in_rust_code_block =
									&*lang == DEFAULT_CODEBLOCK_LANG;
								lang
							}
						}))
					},

					Tag::Link {
						link_type,
						dest_url,
						title,
						id
					} if dest_url.starts_with('#')
						|| link_type == LinkType::Autolink
						|| link_type == LinkType::Email =>
					{
						Tag::Link {
							link_type,
							dest_url,
							title,
							id
						}
					},
					Tag::Link {
						dest_url,
						title,
						id,
						link_type
					} => {
						let link = format!("__link{}", self.link_idx);
						self.link_idx += 1;
						if !dest_url.is_empty() {
							self.links.insert(link.clone(), dest_url.to_string());
						} else if !id.is_empty() {
							self.links.insert(link.clone(), id.to_string());
						} else if !title.is_empty() {
							self.links.insert(link.clone(), title.to_string());
						} else {
							break Some(Event::Start(Tag::Link {
								link_type,
								dest_url,
								title,
								id
							}));
						}
						Tag::Link {
							// pulldown-cmark-to-cmark does not support outputting
							// unresolved reference-style links so we have to do
							// it this stupid way
							link_type: LinkType::Inline,
							dest_url: link.into(),
							title: "".into(),
							id
						}
					},

					// we don't need to modify any other tags
					tag => tag
				}),

				Event::End(tag) => Event::End(match tag {
					// we record when a codeblock ends
					TagEnd::CodeBlock => {
						debug_assert!(
							self.in_code_block,
							"Ending non-started code block, wtf???"
						);
						self.in_code_block = false;
						self.in_rust_code_block = false;
						TagEnd::CodeBlock
					},
					// we don't need to modify any other tags
					tag => tag
				}),

				Event::Text(text) if self.in_code_block && self.in_rust_code_block => {
					let mut filtered = text
						.lines()
						.filter_map(|line| filter_hidden_rust_codeblock_lines(line))
						.join("\n");
					if filtered.is_empty() {
						continue;
					}
					if text.ends_with('\n') {
						filtered.push('\n');
					}
					Event::Text(filtered.into())
				},

				ev => ev
			});
		}
	}
}

struct Readme<'a> {
	template: &'a str,
	builtin_template: bool,
	input: &'a InputFile,

	/// Holds the main markdown part of the readme that was created from the rustdoc,
	/// but does not include any parts of the template or the links.
	readme: String,

	/// Holds the link part of the markdown.
	readme_links: String,

	links: BTreeMap<String, String>
}

impl<'a> Readme<'a> {
	fn new(template: &'a str, builtin_template: bool, input: &'a InputFile) -> Self {
		Self {
			template,
			builtin_template,
			input,
			readme: String::new(),
			readme_links: String::new(),
			links: BTreeMap::new()
		}
	}

	fn write_markdown(&mut self) -> Result<(), pulldown_cmark_to_cmark::Error> {
		// we need this broken link callback for the purpose of broken links being parsed as links
		let mut broken_link_callback = broken_link_callback;
		let parser = Parser::new_with_broken_link_callback(
			&self.input.rustdoc,
			Options::all(),
			Some(&mut broken_link_callback)
		);

		let options = pulldown_cmark_to_cmark::Options {
			code_block_token_count: 3,
			..Default::default()
		};
		pulldown_cmark_to_cmark::cmark_with_options(
			EventFilter::new(parser.into_iter(), &mut self.links),
			&mut self.readme,
			options
		)?;

		// we need to replace the links generated by pulldown-cmark-to-cmark with
		// reference-style links
		let mut i = 0;
		while i < self.readme.len() {
			let Some(idx) = self.readme[i ..].find("(__link") else {
				break;
			};
			let idx = idx + i;
			let Some(idx2) = self.readme[idx ..].find(')') else {
				break;
			};
			let idx2 = idx2 + idx;
			i = idx2;

			self.readme.replace_range(idx ..= idx, "[");
			self.readme.replace_range(idx2 ..= idx2, "]");
		}

		if !self.readme.ends_with('\n') {
			self.readme.push('\n');
		}

		Ok(())
	}

	fn write_links(&mut self) {
		let mut links =
			Links::new(self.template, self.builtin_template, &self.input.rustdoc);
		for link in self.links.keys().cloned().collect::<Vec<_>>() {
			let mut href = self.links[&link].to_owned();
			if href.starts_with('`') && href.ends_with('`') {
				href = href[1 .. href.len() - 1].to_owned();
			}
			let href = self.input.scope.resolve(&self.input.crate_name, href);

			// apply sanitation: If the link looks like a resolved link (i.e. it starts
			// with `::`), and it ends with `()` for functions or `!` for macros, we
			// remove that last token
			let mut href_path = href.path;
			if href_path.starts_with("::") {
				if href_path.ends_with("()") {
					href_path.truncate(href_path.len() - 2);
				} else if href_path.ends_with("!") {
					href_path.truncate(href_path.len() - 1);
				}
			}

			if let Ok(path) = syn::parse_str::<Path>(&href_path) {
				self.links
					.insert(link, links.build_link(&path, href.link_type, self.input));
			} else {
				// debug!("Unable to parse {href_path:?} as syn::Path, not modifying link");
			}
		}

		if !links.deps.is_empty() {
			writeln!(
				self.readme_links,
				" [__cargo_doc2readme_dependencies_info]: {}",
				links.deps.encode()
			)
			.unwrap();
		}
		for (name, href) in &self.links {
			// unwrap: writing to a String never fails
			writeln!(self.readme_links, " [{}]: {}", name, href).unwrap();
		}
	}
}

/// This struct documents all available placeholders in the readme template.
#[derive(Serialize)]
pub struct TemplateContext<'a> {
	/// The name of the crate.
	///
	/// This is renamed to `crate` for the template, do not use ` {{ krate }}`,
	/// it won't work.
	#[serde(rename = "crate")]
	pub krate: &'a str,

	/// The version of the crate.
	///
	/// This is renamed to `crate_version` for the template, do not use `{{ krate_version }}`,
	/// it won't work.
	#[serde(rename = "crate_version")]
	pub krate_version: &'a str,

	/// The target type, etiher `bin` or `lib`.
	pub target: TargetType,

	/// The `repository` variable from the `Cargo.toml` file, if set.
	pub repository: Option<&'a str>,
	/// The HTTP(S) host name of the `repository` variable, if set. This can be used to
	/// add a [Codeberg] icon if the code is hosted on [Codeberg], for example.
	///
	///  [Codeberg]: https://codeberg.org
	pub repository_host: Option<String>,

	/// The `license` variable from the `Cargo.toml` file, if set. Note that in case only
	/// the `license-file` variable is set, this will be none.
	pub license: Option<&'a str>,

	/// The `rust_version` variable from the `Cargo.toml` file, interpreted as a semver
	/// [Version]. This means that, if the `Cargo.toml` specifies `1.90`, this variable
	/// will read `1.90.0`.
	pub rust_version: Option<&'a Version>,

	/// The main content of the readme. Every template should contain this variable
	/// exactly once. Place it whever it is convenient.
	pub readme: String,

	/// The links section of the readme. Place this at the bottom of the readme. You may
	/// omit this if it is empty.
	pub links: String
}

pub fn emit<W: io::Write>(
	input: &InputFile,
	template_filename: &str,
	template: &str,
	builtin_template: bool,
	out_file: W
) -> miette::Result<()> {
	let mut readme = Readme::new(template, builtin_template, input);

	// unwrap: This will never fail since we're only writing to a String.
	// it is just inconvenient to write .unwrap() behind every single write!() invocation
	readme.write_markdown().unwrap();

	readme.write_links();

	let repository = input.repository.as_deref();
	let ctx = TemplateContext {
		krate: &input.crate_name,
		krate_version: &format!("{}", input.crate_version),
		target: input.target_type,
		repository,
		repository_host: repository.and_then(|repo| {
			let url = Url::parse(repo).ok();
			url.as_ref()
				.and_then(|url| url.host_str())
				.map(String::from)
		}),
		license: input.license.as_deref(),
		rust_version: input.rust_version.as_ref(),
		readme: readme.readme,
		links: readme.readme_links
	};

	let mut env = minijinja::Environment::new();
	env.add_template(template_filename, template)
		.map_err(|err| {
			diagnostic::SyntaxError::new_jinja(template_filename, template, err)
		})?;
	env.get_template(template_filename)
		.unwrap()
		.render_to_write(ctx, out_file)
		.into_diagnostic()
		.context("Failed to render template")?;

	Ok(())
}