kellnr_docs/
doc_queue.rs

1use std::path::{Path, PathBuf};
2use std::sync::Arc;
3
4use cargo::GlobalContext;
5use cargo::core::Workspace;
6use cargo::core::resolver::CliFeatures;
7use cargo::ops::{self, CompileOptions, DocOptions, OutputFormat};
8use cargo::util::command_prelude::CompileMode;
9use flate2::read::GzDecoder;
10use fs_extra::dir::{CopyOptions, copy};
11use kellnr_common::original_name::OriginalName;
12use kellnr_common::version::Version;
13use kellnr_db::{DbProvider, DocQueueEntry};
14use kellnr_storage::kellnr_crate_storage::KellnrCrateStorage;
15use tar::Archive;
16use tokio::fs::{create_dir_all, remove_dir_all};
17use tracing::error;
18
19use crate::compute_doc_url;
20use crate::docs_error::DocsError;
21
22pub fn doc_extraction_queue(
23    db: Arc<dyn DbProvider>,
24    cs: Arc<KellnrCrateStorage>,
25    docs_path: PathBuf,
26    path_prefix: String,
27) {
28    tokio::spawn(async move {
29        loop {
30            tokio::time::sleep(std::time::Duration::from_secs(10)).await;
31            if let Err(e) = inner_loop(db.clone(), &cs, &docs_path, &path_prefix).await {
32                error!("Rustdoc generation loop failed: {e}");
33            }
34        }
35    });
36}
37
38async fn inner_loop(
39    db: Arc<dyn DbProvider>,
40    cs: &KellnrCrateStorage,
41    docs_path: &Path,
42    path_prefix: &str,
43) -> Result<(), DocsError> {
44    let entries = db.get_doc_queue().await?;
45
46    for entry in entries {
47        if let Err(e) = extract_docs(&entry, cs, docs_path).await {
48            error!("Failed to extract docs from crate: {e}");
49        } else {
50            if let Err(e) = clean_up(&entry.path).await {
51                error!("Failed to delete temporary rustdoc queue folder: {e}");
52            }
53
54            let version = Version::from_unchecked_str(&entry.version);
55            let docs_link = compute_doc_url(&entry.normalized_name, &version, path_prefix);
56            db.update_docs_link(&entry.normalized_name, &version, &docs_link)
57                .await?;
58        }
59        db.delete_doc_queue(entry.id).await?;
60    }
61
62    Ok(())
63}
64
65async fn extract_docs(
66    doc: &DocQueueEntry,
67    cs: &KellnrCrateStorage,
68    docs_path: &Path,
69) -> Result<(), DocsError> {
70    // Unpack crate
71
72    // TODO: Only works if normalized name = original name -> Need to get original name from db
73    let orig_name = OriginalName::from_unchecked(doc.normalized_name.to_string());
74    let version = Version::from_unchecked_str(&doc.version);
75    let contents = cs.get(&orig_name, &version).await.ok_or_else(|| {
76        error!("Failed to get crate from storage");
77        DocsError::CrateDoesNotExist(doc.normalized_name.to_string(), doc.version.clone())
78    })?;
79    let tar = GzDecoder::new(std::io::Cursor::new(contents));
80    let mut archive = Archive::new(tar);
81    archive.unpack(&doc.path)?;
82
83    // Generate the docs
84    let generated_docs_path = &doc
85        .path
86        .join(format!("{}-{}", doc.normalized_name, doc.version));
87    generate_docs(generated_docs_path)?;
88
89    // Copy the docs directory
90    let from = generated_docs_path.join("target").join("doc");
91    let to = docs_path
92        .join(doc.normalized_name.to_string())
93        .join(&doc.version);
94    copy_dir(&from, &to).await?;
95
96    Ok(())
97}
98
99async fn clean_up(path: &Path) -> Result<(), DocsError> {
100    remove_dir_all(path).await?;
101    Ok(())
102}
103
104async fn copy_dir(from: &Path, to: &Path) -> Result<(), DocsError> {
105    create_dir_all(to).await?;
106    copy(
107        from,
108        to,
109        &CopyOptions {
110            overwrite: true,
111            ..CopyOptions::default()
112        },
113    )?;
114    Ok(())
115}
116
117fn generate_docs(crate_path: impl AsRef<Path>) -> Result<(), DocsError> {
118    let manifest_path = crate_path.as_ref().join("Cargo.toml").canonicalize()?;
119    let ctx = GlobalContext::default().map_err(|e| DocsError::CargoError(e.to_string()))?;
120    let workspace =
121        Workspace::new(&manifest_path, &ctx).map_err(|e| DocsError::CargoError(e.to_string()))?;
122    let compile_opts = CompileOptions {
123        cli_features: CliFeatures::new_all(true),
124        ..CompileOptions::new(
125            &ctx,
126            CompileMode::Doc {
127                deps: false,
128                json: false,
129            },
130        )
131        .map_err(|e| DocsError::CargoError(e.to_string()))?
132    };
133    let options = DocOptions {
134        open_result: false,
135        compile_opts,
136        output_format: OutputFormat::Html,
137    };
138    ops::doc(&workspace, &options).map_err(|e| DocsError::CargoError(e.to_string()))?;
139    Ok(())
140}
kellnr_docs/doc_queue.rs

kellnr_docs/
doc_queue.rs
