doc_chunks/
lib.rs

1//! # Doc Chunks
2//!
3//! `Documentation` is a representation of one or multiple documents.
4//!
5//! A `literal` is a token provided by `proc_macro2` or `ra_ap_syntax` crate, which is then converted by
6//! means of `TrimmedLiteral` using `Cluster`ing into a `CheckableChunk` (mostly
7//! named just `chunk`).
8//!
9//! `CheckableChunk`s can consist of multiple fragments, where each fragment can
10//! span multiple lines, yet each fragment is covering a consecutive `Span` in
11//! the origin content. Each fragment also has a direct mapping to the
12//! `CheckableChunk` internal string representation.
13//!
14//! And `Documentation` holds one or many `CheckableChunks` per file path.
15
16#![deny(unused_crate_dependencies)]
17
18// contains test helpers
19pub mod span;
20pub mod testcase;
21pub use self::span::Span;
22pub use proc_macro2::LineColumn;
23
24pub mod util;
25use self::util::{load_span_from, sub_char_range};
26
27use indexmap::IndexMap;
28use proc_macro2::TokenTree;
29use rayon::prelude::*;
30use serde::Deserialize;
31use std::path::PathBuf;
32use toml::Spanned;
33
34/// Range based on `usize`, simplification.
35pub type Range = core::ops::Range<usize>;
36
37/// Apply an offset to `start` and `end` members, equaling a shift of the range.
38pub fn apply_offset(range: &mut Range, offset: usize) {
39    range.start = range.start.saturating_add(offset);
40    range.end = range.end.saturating_add(offset);
41}
42
43pub mod chunk;
44pub mod cluster;
45mod developer;
46pub mod errors;
47pub mod literal;
48pub mod literalset;
49pub mod markdown;
50
51pub use chunk::*;
52pub use cluster::*;
53pub use errors::*;
54pub use literal::*;
55pub use literalset::*;
56pub use markdown::*;
57
58/// Collection of all the documentation entries across the project
59#[derive(Debug, Clone)]
60pub struct Documentation {
61    /// Mapping of a path to documentation literals
62    index: IndexMap<ContentOrigin, Vec<CheckableChunk>>,
63}
64
65impl Default for Documentation {
66    fn default() -> Self {
67        Self::new()
68    }
69}
70
71impl Documentation {
72    /// Create a new and empty doc.
73    pub fn new() -> Self {
74        Self {
75            index: IndexMap::with_capacity(64),
76        }
77    }
78
79    /// Check if a particular key is contained.
80    pub fn contains_key(&self, key: &ContentOrigin) -> bool {
81        self.index.contains_key(key)
82    }
83
84    /// Check if the document contains any checkable items.
85    #[inline(always)]
86    pub fn is_empty(&self) -> bool {
87        self.index.is_empty()
88    }
89
90    /// Borrowing iterator across content origins and associated sets of chunks.
91    #[inline(always)]
92    pub fn iter(&self) -> impl Iterator<Item = (&ContentOrigin, &Vec<CheckableChunk>)> {
93        self.index.iter()
94    }
95
96    /// Borrowing iterator across content origins and associated sets of chunks.
97    pub fn par_iter(&self) -> impl ParallelIterator<Item = (&ContentOrigin, &Vec<CheckableChunk>)> {
98        self.index.par_iter()
99    }
100
101    /// Consuming iterator across content origins and associated sets of chunks.
102    pub fn into_par_iter(
103        self,
104    ) -> impl ParallelIterator<Item = (ContentOrigin, Vec<CheckableChunk>)> {
105        self.index.into_par_iter()
106    }
107
108    /// Extend `self` by joining in other `Documentation`s.
109    pub fn extend<I, J>(&mut self, other: I)
110    where
111        I: IntoIterator<Item = (ContentOrigin, Vec<CheckableChunk>), IntoIter = J>,
112        J: Iterator<Item = (ContentOrigin, Vec<CheckableChunk>)>,
113    {
114        other
115            .into_iter()
116            .for_each(|(origin, chunks): (_, Vec<CheckableChunk>)| {
117                self.add_inner(origin, chunks);
118            });
119    }
120
121    /// Adds a set of `CheckableChunk`s to the documentation to be checked.
122    pub fn add_inner(&mut self, origin: ContentOrigin, mut chunks: Vec<CheckableChunk>) {
123        self.index
124            .entry(origin)
125            .and_modify(|acc: &mut Vec<CheckableChunk>| {
126                acc.append(&mut chunks);
127            })
128            .or_insert_with(|| chunks);
129        // Ok(()) TODO make this failable
130    }
131
132    /// Adds a rust content str to the documentation.
133    pub fn add_rust(
134        &mut self,
135        origin: ContentOrigin,
136        content: &str,
137        doc_comments: bool,
138        dev_comments: bool,
139    ) -> Result<()> {
140        let cluster = Clusters::load_from_str(content, doc_comments, dev_comments)?;
141
142        let chunks = Vec::<CheckableChunk>::from(cluster);
143        self.add_inner(origin, chunks);
144        Ok(())
145    }
146
147    /// Adds a content string to the documentation sourced from the
148    /// `description` field in a `Cargo.toml` manifest.
149    pub fn add_cargo_manifest_description(
150        &mut self,
151        path: PathBuf,
152        manifest_content: &str,
153    ) -> Result<()> {
154        fn extract_range_of_description(manifest_content: &str) -> Result<Range> {
155            #[derive(Deserialize, Debug)]
156            struct Manifest {
157                package: Spanned<Package>,
158            }
159
160            #[derive(Deserialize, Debug)]
161            struct Package {
162                description: Spanned<String>,
163            }
164
165            let value: Manifest = toml::from_str(manifest_content)?;
166            let d = value.package.into_inner().description;
167            let range = d.span();
168            Ok(range)
169        }
170
171        let mut range = extract_range_of_description(manifest_content)?;
172        let description = sub_char_range(manifest_content, range.clone());
173
174        // Attention: `description` does include `\"\"\"` as well as `\\\n`, the latter is not a big issue,
175        // but the trailing start and end delimiters are.
176        // TODO: split into multiple on `\\\n` and create multiple range/span mappings.
177        let description = if range.len() > 6 {
178            if description.starts_with("\"\"\"") {
179                range.start += 3;
180                range.end -= 3;
181                assert!(!range.is_empty());
182            }
183            dbg!(&description[3..(description.len()) - 3])
184        } else {
185            description
186        };
187
188        fn convert_range_to_span(content: &str, range: Range) -> Option<Span> {
189            let mut line = 0_usize;
190            let mut column = 0_usize;
191            let mut prev = '\n';
192            let mut start = None;
193            for (offset, c) in content.chars().enumerate() {
194                if prev == '\n' {
195                    column = 0;
196                    line += 1;
197                }
198                prev = c;
199
200                if offset == range.start {
201                    start = Some(LineColumn { line, column });
202                    continue;
203                }
204                // take care of inclusivity
205                if offset + 1 == range.end {
206                    let end = LineColumn { line, column };
207                    return Some(Span {
208                        start: start.unwrap(),
209                        end,
210                    });
211                }
212                column += 1;
213            }
214            None
215        }
216
217        let span = convert_range_to_span(manifest_content, range.clone()).expect(
218            "Description is part of the manifest since it was parsed from the same source. qed",
219        );
220        let origin = ContentOrigin::CargoManifestDescription(path);
221        let source_mapping = dbg!(indexmap::indexmap! {
222            range => span
223        });
224        self.add_inner(
225            origin,
226            vec![CheckableChunk::from_str(
227                description,
228                source_mapping,
229                CommentVariant::TomlEntry,
230            )],
231        );
232        Ok(())
233    }
234
235    /// Adds a common mark content str to the documentation.
236    pub fn add_commonmark(&mut self, origin: ContentOrigin, content: &str) -> Result<()> {
237        // extract the full content span and range
238        let start = LineColumn { line: 1, column: 0 };
239        let end = content
240            .lines()
241            .enumerate()
242            .last()
243            .map(|(idx, linecontent)| (idx + 1, linecontent))
244            .map(|(linenumber, linecontent)| LineColumn {
245                line: linenumber,
246                column: linecontent.chars().count().saturating_sub(1),
247            })
248            .ok_or_else(|| {
249                Error::Span(
250                    "Common mark / markdown file does not contain a single line".to_string(),
251                )
252            })?;
253
254        let span = Span { start, end };
255        let source_mapping = indexmap::indexmap! {
256            0..content.chars().count() => span
257        };
258        self.add_inner(
259            origin,
260            vec![CheckableChunk::from_str(
261                content,
262                source_mapping,
263                CommentVariant::CommonMark,
264            )],
265        );
266        Ok(())
267    }
268
269    /// Obtain the set of chunks for a particular origin.
270    #[inline(always)]
271    pub fn get(&self, origin: &ContentOrigin) -> Option<&[CheckableChunk]> {
272        self.index.get(origin).map(AsRef::as_ref)
273    }
274
275    /// Count the number of origins.
276    #[inline(always)]
277    pub fn entry_count(&self) -> usize {
278        self.index.len()
279    }
280
281    /// Load a document from a single string with a defined origin.
282    pub fn load_from_str(
283        origin: ContentOrigin,
284        content: &str,
285        doc_comments: bool,
286        dev_comments: bool,
287    ) -> Self {
288        let mut docs = Documentation::new();
289
290        match origin.clone() {
291            ContentOrigin::RustDocTest(_path, span) => {
292                if let Ok(excerpt) = load_span_from(&mut content.as_bytes(), span) {
293                    docs.add_rust(origin.clone(), excerpt.as_str(), doc_comments, dev_comments)
294                } else {
295                    // TODO
296                    Ok(())
297                }
298            }
299            origin @ ContentOrigin::RustSourceFile(_) => {
300                docs.add_rust(origin, content, doc_comments, dev_comments)
301            }
302            ContentOrigin::CargoManifestDescription(path) => {
303                docs.add_cargo_manifest_description(path, content)
304            }
305            origin @ ContentOrigin::CommonMarkFile(_) => docs.add_commonmark(origin, content),
306            origin @ ContentOrigin::TestEntityRust => {
307                docs.add_rust(origin, content, doc_comments, dev_comments)
308            }
309            origin @ ContentOrigin::TestEntityCommonMark => docs.add_commonmark(origin, content),
310        }
311        .unwrap_or_else(move |e| {
312            log::warn!(
313                "BUG: Failed to load content from {origin} (dev_comments={dev_comments:?}): {e:?}",
314            );
315        });
316        docs
317    }
318
319    pub fn len(&self) -> usize {
320        self.index.len()
321    }
322}
323
324impl IntoIterator for Documentation {
325    type Item = (ContentOrigin, Vec<CheckableChunk>);
326    type IntoIter = indexmap::map::IntoIter<ContentOrigin, Vec<CheckableChunk>>;
327
328    fn into_iter(self) -> Self::IntoIter {
329        self.index.into_iter()
330    }
331}
doc_chunks/lib.rs

doc_chunks/
lib.rs
