kellnr_docs/
doc_queue.rs

1use std::path::{Path, PathBuf};
2use std::sync::Arc;
3
4use cargo::GlobalContext;
5use cargo::core::Workspace;
6use cargo::core::resolver::CliFeatures;
7use cargo::ops::{self, CompileOptions, DocOptions, OutputFormat};
8use cargo::util::command_prelude::CompileMode;
9use flate2::read::GzDecoder;
10use fs_extra::dir::{CopyOptions, copy};
11use kellnr_common::original_name::OriginalName;
12use kellnr_common::version::Version;
13use kellnr_db::{DbProvider, DocQueueEntry};
14use kellnr_storage::kellnr_crate_storage::KellnrCrateStorage;
15use tar::Archive;
16use tokio::fs::{create_dir_all, remove_dir_all};
17use tracing::error;
18
19use crate::compute_doc_url;
20use crate::docs_error::DocsError;
21
22pub fn doc_extraction_queue(
23    db: Arc<dyn DbProvider>,
24    cs: Arc<KellnrCrateStorage>,
25    docs_path: PathBuf,
26) {
27    tokio::spawn(async move {
28        loop {
29            tokio::time::sleep(std::time::Duration::from_secs(10)).await;
30            if let Err(e) = inner_loop(db.clone(), &cs, &docs_path).await {
31                error!("Rustdoc generation loop failed: {e}");
32            }
33        }
34    });
35}
36
37async fn inner_loop(
38    db: Arc<dyn DbProvider>,
39    cs: &KellnrCrateStorage,
40    docs_path: &Path,
41) -> Result<(), DocsError> {
42    let entries = db.get_doc_queue().await?;
43
44    for entry in entries {
45        if let Err(e) = extract_docs(&entry, cs, docs_path).await {
46            error!("Failed to extract docs from crate: {e}");
47        } else {
48            if let Err(e) = clean_up(&entry.path).await {
49                error!("Failed to delete temporary rustdoc queue folder: {e}");
50            }
51
52            let version = Version::from_unchecked_str(&entry.version);
53            let docs_link = compute_doc_url(&entry.normalized_name, &version);
54            db.update_docs_link(&entry.normalized_name, &version, &docs_link)
55                .await?;
56        }
57        db.delete_doc_queue(entry.id).await?;
58    }
59
60    Ok(())
61}
62
63async fn extract_docs(
64    doc: &DocQueueEntry,
65    cs: &KellnrCrateStorage,
66    docs_path: &Path,
67) -> Result<(), DocsError> {
68    // Unpack crate
69
70    // TODO: Only works if normalized name = original name -> Need to get original name from db
71    let orig_name = OriginalName::from_unchecked(doc.normalized_name.to_string());
72    let version = Version::from_unchecked_str(&doc.version);
73    let contents = cs.get(&orig_name, &version).await.ok_or_else(|| {
74        error!("Failed to get crate from storage");
75        DocsError::CrateDoesNotExist(doc.normalized_name.to_string(), doc.version.clone())
76    })?;
77    let tar = GzDecoder::new(std::io::Cursor::new(contents));
78    let mut archive = Archive::new(tar);
79    archive.unpack(&doc.path)?;
80
81    // Generate the docs
82    let generated_docs_path = &doc
83        .path
84        .join(format!("{}-{}", doc.normalized_name, doc.version));
85    generate_docs(generated_docs_path)?;
86
87    // Copy the docs directory
88    let from = generated_docs_path.join("target").join("doc");
89    let to = docs_path
90        .join(doc.normalized_name.to_string())
91        .join(&doc.version);
92    copy_dir(&from, &to).await?;
93
94    Ok(())
95}
96
97async fn clean_up(path: &Path) -> Result<(), DocsError> {
98    remove_dir_all(path).await?;
99    Ok(())
100}
101
102async fn copy_dir(from: &Path, to: &Path) -> Result<(), DocsError> {
103    create_dir_all(to).await?;
104    copy(
105        from,
106        to,
107        &CopyOptions {
108            overwrite: true,
109            ..CopyOptions::default()
110        },
111    )?;
112    Ok(())
113}
114
115fn generate_docs(crate_path: impl AsRef<Path>) -> Result<(), DocsError> {
116    let manifest_path = crate_path.as_ref().join("Cargo.toml").canonicalize()?;
117    let ctx = GlobalContext::default().map_err(|e| DocsError::CargoError(e.to_string()))?;
118    let workspace =
119        Workspace::new(&manifest_path, &ctx).map_err(|e| DocsError::CargoError(e.to_string()))?;
120    let compile_opts = CompileOptions {
121        cli_features: CliFeatures::new_all(true),
122        ..CompileOptions::new(
123            &ctx,
124            CompileMode::Doc {
125                deps: false,
126                json: false,
127            },
128        )
129        .map_err(|e| DocsError::CargoError(e.to_string()))?
130    };
131    let options = DocOptions {
132        open_result: false,
133        compile_opts,
134        output_format: OutputFormat::Html,
135    };
136    ops::doc(&workspace, &options).map_err(|e| DocsError::CargoError(e.to_string()))?;
137    Ok(())
138}
kellnr_docs/doc_queue.rs

kellnr_docs/
doc_queue.rs
