bundle_sources/
lib.rs

1// Copyright 2020 Ian Jackson
2// SPDX-License-Identifier: GPL-3.0-or-later
3// There is NO WARRANTY.
4
5//! Unfortunately, this package is not yet documented.
6
7// BUNDLE_RUST_SOURCES_TEST_OUTPUT=/home/rustcargo/Rustup/Tau/bundle-sources/d rustcargo test
8
9mod imports;
10
11pub mod immutable;
12pub mod utils;
13
14mod bundle;
15mod html;
16
17pub use crate::bundle::*;
18
19use html::*;
20pub use html::{Html,HtmlRef};
21
22use crate::immutable::Immutable;
23use crate::imports::*;
24use crate::utils::*;
25
26#[derive(Error,Debug)]
27pub enum Error {
28  #[error("output leafname {0:?} has bad syntax")]
29  BadLeafBasename(String),
30
31  #[error("output leafname {leafname:?} reused ({first} vs {second})")]
32  LeafReused { leafname : String, first : String, second : String },
33
34  #[error("full leafname {leafname:?} generated twice by same bundle from different basenames, now due to bundle {bundle_id} - bundle id is not unique or seems to generate inconsistent suffix, or something?")]
35  ComponentLeafGeneratedTwiceInconsistency
36  { bundle_id : String, leafname : String },
37}
38
39use Error::*;
40
41pub struct SourceGenerator {
42  output_dir : Immutable<String>,
43  config : Config,
44  items : BTreeMap<Category, Vec<OutputItem>>,
45  used_leaves : HashMap<String, String>,
46}
47
48pub struct Config {
49  include_upstream : bool,
50  cargo_source_cat : Box<dyn FnMut(&Option<Source>) -> Result<Category,E>>,
51  tidy : bool,
52}
53
54impl Default for Config {
55  fn default() -> Config {
56    let cargo_source_cat = Box::new(|s : &Option<Source>| Ok(
57      if s.as_ref().map_or(false, |s : &Source| s.is_crates_io())
58      { Upstream } else { Modified }
59    ));
60
61
62    Config {
63      include_upstream : true,
64      tidy : true,
65      cargo_source_cat,
66    }
67  }
68}
69
70impl Config {
71  pub fn new() -> Config { Default::default() }
72  pub fn include_upstream(&mut self, i : bool) -> &mut Self {
73    self.include_upstream = i; self
74  }
75  pub fn tidy(&mut self, i : bool) -> &mut Self {
76    self.tidy = i; self
77  }
78}
79
80#[derive(Debug,Clone,Serialize)]
81pub struct OutputItem {
82  pub leafname : String,
83  pub desc : Html,
84  cat : Category,
85}
86
87#[derive(Debug,Ord,PartialOrd,Eq,PartialEq,Copy,Clone,Hash)]
88#[derive(Serialize,Deserialize)]
89pub enum Category {
90  Upstream,
91  Mixed,
92  Modified,
93  Metadata,
94}
95use Category::*;
96
97impl SourceGenerator {
98  //#[throws(E)]
99  pub fn new_adding(output_dir : String) -> Result<SourceGenerator,E> {
100    let output_dir = output_dir.to_owned();
101    match create_dir(&output_dir) {
102      Ok(_)                               => (),
103      Err(e) if e.kind() == AlreadyExists => (),
104      e @ Err(_) => e.with_context(|| format!(
105        "new_adding: create output directory, if needed {:?}",
106        &output_dir
107      ))?
108    };
109    Ok( Self::new_return(output_dir) )
110  }
111
112  #[throws(E)]
113  pub fn new_trashing(output_dir : String) -> SourceGenerator {
114    let output_dir = output_dir.to_owned();
115    match remove_dir_all(&output_dir) {
116      Ok(_) => { },
117      Err(e) if e.kind() == NotFound => { },
118      e @ Err(_) => 
119        e.cxm(||format!(
120          "new_trashing: remove original output directory {:?}",
121          &output_dir))?,
122    }
123    create_dir(&output_dir).cxm(||format!(
124      "new_trashing: create new output directory {:?}",
125      &output_dir))?;
126    Self::new_return(output_dir)
127  }
128
129  fn new_return(output_dir : String) -> SourceGenerator {
130    let output_dir = output_dir.into();
131    let items = Default::default();
132    let used_leaves = HashMap::new();
133    let config = Config::new();
134    SourceGenerator { output_dir, items, used_leaves, config }
135  }
136  pub fn configure(&mut self, config : Config) -> &mut Self {
137    self.config = config;
138    self
139  }
140
141  fn output_file<T : AsRef<str>>(&self, sub : T) -> String {
142    self.output_dir.to_owned() + "/" + sub.as_ref()
143  }
144
145  #[throws(E)]
146  /// `desc` should be a single line of text in HTML syntax
147  pub fn add_component<C : Component>(&mut self,
148            src : &C, basename : &str,
149            cat : Category, desc : Html)
150  {
151    let basename = basename.to_owned();
152    let baseleaf = basename; // makes basename unuseable
153    let bundle_id = format!("{:?} {:?}", cat, src);
154    if self.check_record_leaf_done(&baseleaf, &bundle_id)? { return }
155
156    let basepath = self.output_file(&baseleaf);
157    let suffix = src.bundle(&basepath)?;
158    let already_checked = suffix.is_none();
159
160    let suffix = suffix.as_ref().map(|s| s.as_ref()).unwrap_or("");
161    let fullleaf = baseleaf + suffix;
162    self.add_bundle(fullleaf, cat, desc, bundle_id, already_checked)?;
163  }
164
165  #[throws(Error)]
166  fn check_record_leaf_done(&mut self, leafname : &str, bundle_id : &str)
167                            -> bool {
168    let leafname = leafname.to_owned();
169    let bundle_id = bundle_id.to_owned();
170    if leafname.len() == 0
171      || !leafname.starts_with(|c:char| c.is_ascii_lowercase())
172      || leafname.find('/').is_some()
173      || leafname.split('.').next().unwrap() == INDEX_BASENAME {
174        throw!(BadLeafBasename(leafname))
175      }
176    match self.used_leaves.entry(leafname) {
177      Vacant(e) => {
178        e.insert(bundle_id);
179        false
180      }
181      Occupied(e) if e.get() == &bundle_id => {
182        true
183      }
184      Occupied(e) => {
185        throw!(LeafReused{
186          leafname : e.key().clone(),
187          first : e.get().clone(),
188          second : bundle_id,
189        })
190      }
191    }
192  }
193
194  #[throws(Error)]
195  fn add_bundle(&mut self, fullleaf : String,
196                cat : Category, desc : Html, bundle_id : String,
197                already_checked: bool) {
198    if !already_checked {
199      let already = self.check_record_leaf_done(&fullleaf, &bundle_id)?;
200      if already { throw!(ComponentLeafGeneratedTwiceInconsistency{
201        bundle_id, leafname : fullleaf }) }
202    }
203    self.items.entry(cat).or_default().push(
204      OutputItem { leafname : fullleaf, desc, cat });
205  }
206
207  #[throws(E)]
208  /// Some("Cargo.toml") mentions the package name for the metadata json
209  pub fn add_cargo_packages(&mut self, spec_manifest_path : Option<&str>) {
210    let manifest_path = match spec_manifest_path
211      .map( |i| i.to_owned() )
212    {
213      Some(i) if i == "Cargo.toml" || i.ends_with("/Cargo.toml") => i,
214      Some(i) => { i + "/Cargo.toml" },
215      None => {
216        match env::var("CARGO_MANIFEST_DIR") {
217          Err(VarError::NotPresent) => "Cargo.toml".to_owned(),
218          e @ Err(_) => e.cxm(||"looking up CARGO_MANIFEST_DIR".to_owned())?,
219          Ok(d) => d + "/Cargo.toml",
220        }
221      }
222    };
223
224    #[throws(E)]
225    fn package_leafname(name : &str, version : &str) -> String {
226      let check = |s : &str, what|{
227        let badness = s.trim_matches(
228          |c : char| c=='.' || c=='_' || c=='-' || c=='+' ||
229                     c.is_ascii_alphanumeric()
230        );
231        if badness.len() != 0 {
232          Err(anyhow!("package {:?} {} bad syntax ({:?})",
233                      name, what, badness))?;
234        }
235        <Result<(),E>>::Ok(())
236      };
237      check(version,"version")?;
238      check(name,"name")?;
239      format!("{}={}", name, version)
240    }
241  
242    let (metafile_leafname, metafile_desc) = if spec_manifest_path.is_none() {
243      ("cargo-metadata.json".to_owned(),
244       Html("Rust cargo metadatafile".to_owned()))
245    } else {
246      (||{
247        let manifest_data = fs::read_to_string(&manifest_path)?;
248        let manifest_data = manifest_data.parse()?;
249        let getkey
250          : for<'v> fn(&'v toml::value::Value,&'_ str) -> Result<&'v _,E>
251          = |d,k| {
252            d.get(k).ok_or_else(
253              ||anyhow!(r#"missing "{}" in Cargo.toml"#, k))
254          };
255        let getstr = |d, k| {
256          getkey(d,k)?.as_str().ok_or_else(
257            ||anyhow!(r#"value for "{}" is not a string"#, k))
258        };
259        let package = getkey(&manifest_data,"package")?;
260        let name = getstr(package,"name")?;
261        let version = getstr(package,"version")?;
262        let metafile_base = package_leafname(&name, &version)?;
263        <Result<_,E>>::Ok((
264          metafile_base + ".cargo-metadata.json",
265          Html(format!("Rust cargo metadatafile for package {}",
266                       Html::from_literal(&name).0))
267        ))
268      })().cxm(||format!("read manifest {:?} to find package name and version",
269                         manifest_path))?
270    };
271
272    let metafile_path = self.output_file(&metafile_leafname);
273
274    let mut cmd = MetadataCommand::new()
275      .manifest_path(&manifest_path)
276      .other_options(vec!["--offline".to_string()])
277      .other_options(vec!["--locked".to_string()])
278      .cargo_command();
279    cmd
280      .stdout(File::create(&metafile_path)?);
281    if let Some(cd) = env::var_os("CARGO_HOME") {
282      cmd.current_dir(cd);
283    }
284    let st = cmd
285      .status()?;
286
287    let bundle_id = format!("add_cargo_packages(manifest_path={:?})",
288                            &manifest_path);
289
290    self.add_bundle(metafile_leafname, Metadata,
291                    metafile_desc, bundle_id, false)?;
292
293    check_exit_status(st).context("cargo metadata failed")?;
294
295    let metadata = fs::read_to_string(metafile_path)
296      .context("cannot read metadata file")?;
297    let mut metadata = MetadataCommand::parse(&metadata)?;
298
299    metadata.packages.sort_by(|a,b|{
300      let gf : fn(&_) -> (&_,&_,&_) =
301        |p : &Package| (&p.name, &p.version, &p.id);
302      gf(a).cmp(&gf(b))
303    });
304    for package in &metadata.packages {
305      let pmf = &package.manifest_path;
306      let src = pmf.parent().ok_or_else(
307        || anyhow!("manifest path without directory: {:?}", &package)
308      )?.as_str();
309      let outleaf = package_leafname(&package.name,
310                                     &format!("{}",&package.version))?;
311      let desc = Html(format!(
312        "rust package {} version {}",
313        Html::from_literal(&package.name).0,
314        Html::from_literal(&format!("{}",package.version)).0));
315
316      let cat = (self.config.cargo_source_cat)(&package.source)?;
317      self.add_component(&DirectoryComponent::new(src.to_owned())?,
318                         &outleaf, cat, desc)?;
319    }
320  }
321
322  #[throws(E)]
323  pub fn aggregate(mut self) -> Vec<OutputItem> {
324    let mut categories : Vec<_> = self.items.keys().map(|c| *c).collect();
325    categories.sort();
326    categories.reverse();
327
328    let mut results = vec![];
329    {
330      let include_upstream = self.config.include_upstream;
331      let mut agg = |mincat, nomcat, outleaf, desc |{
332        self.aggregate_min_category(&categories,&mut results,
333                                    mincat,nomcat,outleaf,desc)
334          .cxm(||format!("failure aggregating {}", outleaf))
335      };
336      agg(Mixed, Modified, "Modified",
337          &HtmlRef("Sources, locally modified"))?;
338      if include_upstream {
339        agg(Upstream, Mixed, "Comprehensive",
340            &HtmlRef("All sources"))?;
341      }
342    }
343
344    let mut index = IndexWriter::new(&self.output_dir,
345                                     &HtmlRef("Master index"))?;
346    {
347      let mut table = index.table(Html("Source code bundles".to_owned()))?;
348      
349      for &OutputItem { leafname : ref tarfile, ref desc, .. } in &results {
350        table.entry(tarfile, &desc.as_ref())?;
351      }
352    }
353    index.finish()?;
354
355    if self.config.tidy {
356      for OutputItem { ref leafname, .. } in self.items.values().flatten() {
357        remove_file(self.output_file(leafname)).cxm(||format!(
358          "failed to remove {:?} while tidying up", leafname
359        ))?;
360      }
361    }
362
363    (||{
364      let mut f = File::create(self.output_file("index.json"))
365        .context("create output file")?;
366      writeln!(f,"{}",&serde_json::to_string(&results)?)?;
367      f.flush()?;
368      <Result<(),E>>::Ok(())
369    })().context("create index.json file")?;
370
371    let mut output = vec![ OutputItem{
372      leafname : INDEX.to_owned(),
373      cat : Metadata,
374      desc : Html("index of output bundles".to_owned())
375    }];
376    output.extend(results);
377    output
378  }
379
380  #[throws(E)]
381  fn aggregate_min_category(&mut self, cats : &Vec<Category>,
382                            results : &mut Vec<OutputItem>,
383                            mincat : Category, nomcat : Category,
384                            outleafbase : &str, desc : &HtmlRef) {
385    let tarfile = outleafbase.to_string() + ".tar.gz";
386
387    let linkpath = self.output_file(&outleafbase);
388    remove_file(&linkpath).or_else(
389      |e| if e.kind() == NotFound { Ok(()) } else { Err(e) })?;
390    symlink(".", &linkpath)?;
391
392    let mut leaves = vec![ INDEX.to_owned() ];
393    let mut index = IndexWriter::new(&self.output_dir,desc)?;
394
395    for &cat in cats {
396      let mut table = index.table(match cat {
397        Metadata => HtmlRef("Metadata and index files"),
398        Modified => HtmlRef("Local sources, possibly modified"),
399        Upstream => HtmlRef("Upstream sources included for completeness"),
400        Mixed    => HtmlRef("Mixed, other or unknown content"),
401      }.to_owned())?;
402      for &OutputItem { ref leafname, cat, ref desc } in &self.items[&cat] {
403        if cat < mincat { continue }
404            leaves.push(leafname.to_owned());
405        table.entry(&leafname, &desc.as_ref())?;
406      }
407    }
408    index.finish()?;
409
410    let mut cmd = tar_command();
411    cmd
412      .current_dir(&*self.output_dir)
413      .args(&["--use-compress-program=gzip --rsyncable",
414              "--no-recursion",
415              "-cf", &tarfile, "--"]);
416    cmd
417      .args( leaves.drain(0..).map(
418        |l| outleafbase.to_owned() + "/" + &l
419      ) );
420    
421    run_cmd(cmd).cxm(||format!("build aggregate (tar) {:?}", &desc))?;
422
423    remove_file(linkpath)?;
424
425    results.push(OutputItem {
426      leafname : tarfile,
427      desc : desc.to_owned(),
428      cat : nomcat,
429    });
430  }
431}
432
433#[cfg(test)] use tempfile::TempDir;
434
435#[test]
436fn selftest() -> Result<(),E> {
437//  use std::path::{Path,PathBuf};
438
439  let output_dir = env::var_os("BUNDLE_RUST_SOURCES_TEST_OUTPUT");
440  let mut _drop_temp = None;
441  let output_dir = match output_dir {
442    None => {
443      _drop_temp = Some(TempDir::new()?);
444      _drop_temp.as_ref().unwrap().path().as_os_str()
445    }
446    Some(ref output_dir) => output_dir,
447  };
448  let output_dir = output_dir.to_str().ok_or_else(
449    || anyhow!("tempdir to_str!")
450  )?;
451  let mut sg = SourceGenerator::new_adding(output_dir.to_string())?;
452  sg.add_cargo_packages(None)?;
453  let _r = sg.aggregate()?;
454  Ok(())
455}
bundle_sources/lib.rs

bundle_sources/
lib.rs
